2024-11-15 23:58:23,921 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f 2024-11-15 23:58:23,936 main DEBUG Took 0.012966 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 23:58:23,937 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 23:58:23,937 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 23:58:23,938 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 23:58:23,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,949 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 23:58:23,965 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,972 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,973 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,974 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,975 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,977 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:58:23,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,979 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 23:58:23,981 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:58:23,983 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 23:58:23,985 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 23:58:23,986 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 23:58:23,987 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 23:58:23,988 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 23:58:23,999 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 23:58:24,002 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 23:58:24,004 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 23:58:24,005 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 23:58:24,005 main DEBUG createAppenders(={Console}) 2024-11-15 23:58:24,006 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f initialized 2024-11-15 23:58:24,007 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f 2024-11-15 23:58:24,007 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5340477f OK. 2024-11-15 23:58:24,008 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 23:58:24,008 main DEBUG OutputStream closed 2024-11-15 23:58:24,008 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 23:58:24,009 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 23:58:24,009 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4659191b OK 2024-11-15 23:58:24,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 23:58:24,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 23:58:24,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 23:58:24,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 23:58:24,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 23:58:24,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 23:58:24,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 23:58:24,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 23:58:24,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 23:58:24,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 23:58:24,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 23:58:24,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 23:58:24,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 23:58:24,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 23:58:24,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 23:58:24,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 23:58:24,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 23:58:24,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 23:58:24,117 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 23:58:24,117 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@30af5b6b) with optional ClassLoader: null 2024-11-15 23:58:24,118 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 23:58:24,118 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@30af5b6b] started OK. 2024-11-15T23:58:24,447 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b 2024-11-15 23:58:24,452 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 23:58:24,453 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T23:58:24,470 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T23:58:24,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T23:58:24,518 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f, deleteOnExit=true 2024-11-15T23:58:24,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T23:58:24,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/test.cache.data in system properties and HBase conf 2024-11-15T23:58:24,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T23:58:24,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.log.dir in system properties and HBase conf 2024-11-15T23:58:24,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T23:58:24,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T23:58:24,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T23:58:24,644 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T23:58:24,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T23:58:24,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T23:58:24,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T23:58:24,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T23:58:24,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T23:58:24,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T23:58:24,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T23:58:24,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T23:58:24,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T23:58:24,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T23:58:24,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/nfs.dump.dir in system properties and HBase conf 2024-11-15T23:58:24,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/java.io.tmpdir in system properties and HBase conf 2024-11-15T23:58:24,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T23:58:24,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T23:58:24,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T23:58:25,783 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T23:58:25,865 INFO [Time-limited test {}] log.Log(170): Logging initialized @2871ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T23:58:25,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:58:26,076 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T23:58:26,145 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T23:58:26,145 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T23:58:26,147 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T23:58:26,169 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:58:26,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.log.dir/,AVAILABLE} 2024-11-15T23:58:26,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T23:58:26,443 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/java.io.tmpdir/jetty-localhost-37205-hadoop-hdfs-3_4_1-tests_jar-_-any-18324422751551085211/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T23:58:26,450 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:37205} 2024-11-15T23:58:26,451 INFO [Time-limited test {}] server.Server(415): Started @3458ms 2024-11-15T23:58:26,973 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T23:58:26,980 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T23:58:26,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T23:58:26,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T23:58:26,984 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T23:58:26,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.log.dir/,AVAILABLE} 2024-11-15T23:58:26,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T23:58:27,153 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15254365{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/java.io.tmpdir/jetty-localhost-45579-hadoop-hdfs-3_4_1-tests_jar-_-any-7271123734050935369/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T23:58:27,155 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ae5a080{HTTP/1.1, (http/1.1)}{localhost:45579} 2024-11-15T23:58:27,156 INFO [Time-limited test {}] server.Server(415): Started @4163ms 2024-11-15T23:58:27,233 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T23:58:27,777 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/data/data1/current/BP-1500091864-172.17.0.3-1731715105508/current, will proceed with Du for space computation calculation, 2024-11-15T23:58:27,778 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/data/data2/current/BP-1500091864-172.17.0.3-1731715105508/current, will proceed with Du for space computation calculation, 2024-11-15T23:58:27,846 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T23:58:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x961fbe0a2551981b with lease ID 0x416f34c68550ee22: Processing first storage report for DS-3ff48f02-c27b-4ddf-8fcb-d3dce79566bf from datanode DatanodeRegistration(127.0.0.1:43069, datanodeUuid=674630d6-a271-46f8-906d-fa5228c2c352, infoPort=35573, infoSecurePort=0, ipcPort=36591, storageInfo=lv=-57;cid=testClusterID;nsid=2056965873;c=1731715105508) 2024-11-15T23:58:27,932 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x961fbe0a2551981b with lease ID 0x416f34c68550ee22: from storage DS-3ff48f02-c27b-4ddf-8fcb-d3dce79566bf node DatanodeRegistration(127.0.0.1:43069, datanodeUuid=674630d6-a271-46f8-906d-fa5228c2c352, infoPort=35573, infoSecurePort=0, ipcPort=36591, storageInfo=lv=-57;cid=testClusterID;nsid=2056965873;c=1731715105508), blocks: 0, hasStaleStorage: true, processing time: 7 msecs, invalidatedBlocks: 0 2024-11-15T23:58:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x961fbe0a2551981b with lease ID 0x416f34c68550ee22: Processing first storage report for DS-208a24de-e240-4496-a369-3a56bae31128 from datanode DatanodeRegistration(127.0.0.1:43069, datanodeUuid=674630d6-a271-46f8-906d-fa5228c2c352, infoPort=35573, infoSecurePort=0, ipcPort=36591, storageInfo=lv=-57;cid=testClusterID;nsid=2056965873;c=1731715105508) 2024-11-15T23:58:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x961fbe0a2551981b with lease ID 0x416f34c68550ee22: from storage DS-208a24de-e240-4496-a369-3a56bae31128 node DatanodeRegistration(127.0.0.1:43069, datanodeUuid=674630d6-a271-46f8-906d-fa5228c2c352, infoPort=35573, infoSecurePort=0, ipcPort=36591, storageInfo=lv=-57;cid=testClusterID;nsid=2056965873;c=1731715105508), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T23:58:27,971 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b 2024-11-15T23:58:28,096 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/zookeeper_0, clientPort=61510, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T23:58:28,116 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61510 2024-11-15T23:58:28,132 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:28,136 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:28,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741825_1001 (size=7) 2024-11-15T23:58:28,863 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e with version=8 2024-11-15T23:58:28,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/hbase-staging 2024-11-15T23:58:28,984 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T23:58:29,281 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0801813e30ae:0 server-side Connection retries=45 2024-11-15T23:58:29,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:29,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:29,300 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T23:58:29,300 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:29,301 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T23:58:29,507 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T23:58:29,581 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T23:58:29,595 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T23:58:29,600 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:58:29,661 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22707 (auto-detected) 2024-11-15T23:58:29,663 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T23:58:29,709 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38967 2024-11-15T23:58:29,751 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38967 connecting to ZooKeeper ensemble=127.0.0.1:61510 2024-11-15T23:58:29,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389670x0, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T23:58:29,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38967-0x100f8d681700000 connected 2024-11-15T23:58:29,946 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:29,949 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:29,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:58:29,973 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e, hbase.cluster.distributed=false 2024-11-15T23:58:30,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T23:58:30,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38967 2024-11-15T23:58:30,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38967 2024-11-15T23:58:30,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38967 2024-11-15T23:58:30,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38967 2024-11-15T23:58:30,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38967 2024-11-15T23:58:30,149 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0801813e30ae:0 server-side Connection retries=45 2024-11-15T23:58:30,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:30,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:30,153 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T23:58:30,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T23:58:30,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T23:58:30,157 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T23:58:30,161 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:58:30,162 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43013 2024-11-15T23:58:30,164 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43013 connecting to ZooKeeper ensemble=127.0.0.1:61510 2024-11-15T23:58:30,166 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:30,169 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:30,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430130x0, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T23:58:30,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430130x0, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T23:58:30,185 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43013-0x100f8d681700001 connected 2024-11-15T23:58:30,191 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T23:58:30,204 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T23:58:30,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T23:58:30,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T23:58:30,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-15T23:58:30,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43013 2024-11-15T23:58:30,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43013 2024-11-15T23:58:30,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-15T23:58:30,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-15T23:58:30,236 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0801813e30ae:38967 2024-11-15T23:58:30,237 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0801813e30ae,38967,1731715109050 2024-11-15T23:58:30,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:58:30,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:58:30,247 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0801813e30ae,38967,1731715109050 2024-11-15T23:58:30,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T23:58:30,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:30,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:30,282 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T23:58:30,284 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0801813e30ae,38967,1731715109050 from backup master directory 2024-11-15T23:58:30,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0801813e30ae,38967,1731715109050 2024-11-15T23:58:30,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:58:30,292 WARN [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T23:58:30,294 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0801813e30ae,38967,1731715109050 2024-11-15T23:58:30,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T23:58:30,297 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T23:58:30,300 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T23:58:30,410 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/hbase.id] with ID: 774650c6-d881-48c6-afec-cf67c8e46ad4 2024-11-15T23:58:30,410 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/.tmp/hbase.id 2024-11-15T23:58:30,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741826_1002 (size=42) 2024-11-15T23:58:30,826 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/.tmp/hbase.id]:[hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/hbase.id] 2024-11-15T23:58:30,909 INFO [master/0801813e30ae:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T23:58:30,916 INFO [master/0801813e30ae:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T23:58:30,945 INFO [master/0801813e30ae:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-15T23:58:30,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:30,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:30,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741827_1003 (size=196) 2024-11-15T23:58:31,006 INFO [master/0801813e30ae:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:58:31,012 INFO [master/0801813e30ae:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T23:58:31,037 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:31,044 INFO [master/0801813e30ae:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:58:31,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741828_1004 (size=1189) 2024-11-15T23:58:31,522 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store 2024-11-15T23:58:31,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741829_1005 (size=34) 2024-11-15T23:58:31,955 INFO [master/0801813e30ae:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T23:58:31,959 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:31,961 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T23:58:31,961 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:58:31,961 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:58:31,963 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T23:58:31,964 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:58:31,964 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T23:58:31,965 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731715111961Disabling compacts and flushes for region at 1731715111961Disabling writes for close at 1731715111963 (+2 ms)Writing region close event to WAL at 1731715111964 (+1 ms)Closed at 1731715111964 2024-11-15T23:58:31,968 WARN [master/0801813e30ae:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/.initializing 2024-11-15T23:58:31,969 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/WALs/0801813e30ae,38967,1731715109050 2024-11-15T23:58:31,980 INFO [master/0801813e30ae:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T23:58:32,000 INFO [master/0801813e30ae:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0801813e30ae%2C38967%2C1731715109050, suffix=, logDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/WALs/0801813e30ae,38967,1731715109050, archiveDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/oldWALs, maxLogs=10 2024-11-15T23:58:32,030 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/WALs/0801813e30ae,38967,1731715109050/0801813e30ae%2C38967%2C1731715109050.1731715112005, exclude list is [], retry=0 2024-11-15T23:58:32,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43069,DS-3ff48f02-c27b-4ddf-8fcb-d3dce79566bf,DISK] 2024-11-15T23:58:32,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T23:58:32,099 INFO [master/0801813e30ae:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/WALs/0801813e30ae,38967,1731715109050/0801813e30ae%2C38967%2C1731715109050.1731715112005 2024-11-15T23:58:32,100 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35573:35573)] 2024-11-15T23:58:32,101 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:58:32,101 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:32,104 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,105 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T23:58:32,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:32,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:32,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T23:58:32,183 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:32,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:32,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T23:58:32,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:32,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:32,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T23:58:32,193 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:32,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:32,195 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,199 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,200 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,205 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,206 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,211 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T23:58:32,215 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T23:58:32,221 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:58:32,222 INFO [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64422594, jitterRate=-0.040028542280197144}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T23:58:32,227 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731715112118Initializing all the Stores at 1731715112120 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715112121 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715112122 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715112122Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715112122Cleaning up temporary data from old regions at 1731715112206 (+84 ms)Region opened successfully at 1731715112227 (+21 ms) 2024-11-15T23:58:32,229 INFO [master/0801813e30ae:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T23:58:32,273 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5774c48e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0801813e30ae/172.17.0.3:0 2024-11-15T23:58:32,317 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T23:58:32,332 INFO [master/0801813e30ae:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T23:58:32,332 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T23:58:32,336 INFO [master/0801813e30ae:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T23:58:32,337 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T23:58:32,343 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T23:58:32,343 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T23:58:32,392 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T23:58:32,404 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T23:58:32,407 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T23:58:32,411 INFO [master/0801813e30ae:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T23:58:32,412 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T23:58:32,415 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T23:58:32,418 INFO [master/0801813e30ae:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T23:58:32,425 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T23:58:32,427 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T23:58:32,429 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T23:58:32,433 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T23:58:32,469 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T23:58:32,473 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T23:58:32,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T23:58:32,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T23:58:32,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,482 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0801813e30ae,38967,1731715109050, sessionid=0x100f8d681700000, setting cluster-up flag (Was=false) 2024-11-15T23:58:32,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,507 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T23:58:32,514 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0801813e30ae,38967,1731715109050 2024-11-15T23:58:32,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:32,532 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T23:58:32,534 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0801813e30ae,38967,1731715109050 2024-11-15T23:58:32,542 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T23:58:32,634 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(746): ClusterId : 774650c6-d881-48c6-afec-cf67c8e46ad4 2024-11-15T23:58:32,642 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T23:58:32,651 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T23:58:32,651 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T23:58:32,652 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T23:58:32,655 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T23:58:32,656 DEBUG [RS:0;0801813e30ae:43013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620e598c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0801813e30ae/172.17.0.3:0 2024-11-15T23:58:32,665 INFO [master/0801813e30ae:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T23:58:32,674 INFO [master/0801813e30ae:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T23:58:32,678 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0801813e30ae:43013 2024-11-15T23:58:32,680 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0801813e30ae,38967,1731715109050 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T23:58:32,685 INFO [RS:0;0801813e30ae:43013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T23:58:32,685 INFO [RS:0;0801813e30ae:43013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T23:58:32,685 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T23:58:32,688 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(2659): reportForDuty to master=0801813e30ae,38967,1731715109050 with port=43013, startcode=1731715110098 2024-11-15T23:58:32,690 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0801813e30ae:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:58:32,690 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0801813e30ae:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:58:32,690 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0801813e30ae:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:58:32,691 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0801813e30ae:0, corePoolSize=5, maxPoolSize=5 2024-11-15T23:58:32,691 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0801813e30ae:0, corePoolSize=10, maxPoolSize=10 2024-11-15T23:58:32,691 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,692 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0801813e30ae:0, corePoolSize=2, maxPoolSize=2 2024-11-15T23:58:32,692 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,699 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731715142699 2024-11-15T23:58:32,700 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:58:32,701 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T23:58:32,701 DEBUG [RS:0;0801813e30ae:43013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T23:58:32,701 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T23:58:32,702 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T23:58:32,706 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T23:58:32,706 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T23:58:32,707 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T23:58:32,707 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T23:58:32,708 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:32,709 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T23:58:32,713 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,717 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T23:58:32,721 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T23:58:32,721 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T23:58:32,725 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T23:58:32,725 INFO [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T23:58:32,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741831_1007 (size=1321) 2024-11-15T23:58:32,736 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.large.0-1731715112727,5,FailOnTimeoutGroup] 2024-11-15T23:58:32,741 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.small.0-1731715112737,5,FailOnTimeoutGroup] 2024-11-15T23:58:32,742 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,742 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T23:58:32,743 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,743 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T23:58:32,756 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38967 {}] master.ServerManager(517): Registering regionserver=0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,778 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-15T23:58:32,778 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44829 2024-11-15T23:58:32,778 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T23:58:32,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T23:58:32,783 DEBUG [RS:0;0801813e30ae:43013 {}] zookeeper.ZKUtil(111): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,783 WARN [RS:0;0801813e30ae:43013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T23:58:32,784 INFO [RS:0;0801813e30ae:43013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:58:32,784 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,786 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0801813e30ae,43013,1731715110098] 2024-11-15T23:58:32,814 INFO [RS:0;0801813e30ae:43013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T23:58:32,835 INFO [RS:0;0801813e30ae:43013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T23:58:32,845 INFO [RS:0;0801813e30ae:43013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T23:58:32,845 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,846 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T23:58:32,853 INFO [RS:0;0801813e30ae:43013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T23:58:32,855 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,855 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,855 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,855 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,856 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,856 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,856 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0801813e30ae:0, corePoolSize=2, maxPoolSize=2 2024-11-15T23:58:32,856 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,856 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0801813e30ae:0, corePoolSize=1, maxPoolSize=1 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0801813e30ae:0, corePoolSize=3, maxPoolSize=3 2024-11-15T23:58:32,857 DEBUG [RS:0;0801813e30ae:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0, corePoolSize=3, maxPoolSize=3 2024-11-15T23:58:32,859 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,859 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,859 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,859 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,860 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,860 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,43013,1731715110098-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T23:58:32,890 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T23:58:32,893 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,43013,1731715110098-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,894 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,894 INFO [RS:0;0801813e30ae:43013 {}] regionserver.Replication(171): 0801813e30ae,43013,1731715110098 started 2024-11-15T23:58:32,917 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:32,917 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1482): Serving as 0801813e30ae,43013,1731715110098, RpcServer on 0801813e30ae/172.17.0.3:43013, sessionid=0x100f8d681700001 2024-11-15T23:58:32,918 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T23:58:32,918 DEBUG [RS:0;0801813e30ae:43013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,918 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0801813e30ae,43013,1731715110098' 2024-11-15T23:58:32,918 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T23:58:32,920 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T23:58:32,921 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T23:58:32,921 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T23:58:32,921 DEBUG [RS:0;0801813e30ae:43013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0801813e30ae,43013,1731715110098 2024-11-15T23:58:32,921 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0801813e30ae,43013,1731715110098' 2024-11-15T23:58:32,921 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T23:58:32,922 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T23:58:32,923 DEBUG [RS:0;0801813e30ae:43013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T23:58:32,923 INFO [RS:0;0801813e30ae:43013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T23:58:32,923 INFO [RS:0;0801813e30ae:43013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T23:58:33,031 INFO [RS:0;0801813e30ae:43013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T23:58:33,036 INFO [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0801813e30ae%2C43013%2C1731715110098, suffix=, logDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098, archiveDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/oldWALs, maxLogs=32 2024-11-15T23:58:33,063 DEBUG [RS:0;0801813e30ae:43013 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098/0801813e30ae%2C43013%2C1731715110098.1731715113040, exclude list is [], retry=0 2024-11-15T23:58:33,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43069,DS-3ff48f02-c27b-4ddf-8fcb-d3dce79566bf,DISK] 2024-11-15T23:58:33,086 INFO [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098/0801813e30ae%2C43013%2C1731715110098.1731715113040 2024-11-15T23:58:33,087 DEBUG [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35573:35573)] 2024-11-15T23:58:33,130 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T23:58:33,130 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-15T23:58:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741833_1009 (size=32) 2024-11-15T23:58:33,567 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:33,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T23:58:33,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T23:58:33,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:33,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:33,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T23:58:33,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T23:58:33,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:33,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:33,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T23:58:33,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T23:58:33,585 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:33,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:33,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T23:58:33,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T23:58:33,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:33,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:33,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T23:58:33,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740 2024-11-15T23:58:33,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740 2024-11-15T23:58:33,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T23:58:33,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T23:58:33,597 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:58:33,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T23:58:33,604 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:58:33,605 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68981157, jitterRate=0.02789933979511261}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:58:33,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731715113567Initializing all the Stores at 1731715113570 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715113570Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715113572 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715113572Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715113573 (+1 ms)Cleaning up temporary data from old regions at 1731715113596 (+23 ms)Region opened successfully at 1731715113607 (+11 ms) 2024-11-15T23:58:33,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T23:58:33,608 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T23:58:33,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T23:58:33,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T23:58:33,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T23:58:33,609 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T23:58:33,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731715113607Disabling compacts and flushes for region at 1731715113607Disabling writes for close at 1731715113608 (+1 ms)Writing region close event to WAL at 1731715113609 (+1 ms)Closed at 1731715113609 2024-11-15T23:58:33,613 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:58:33,613 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T23:58:33,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T23:58:33,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T23:58:33,636 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T23:58:33,790 DEBUG [0801813e30ae:38967 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T23:58:33,807 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:58:33,815 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0801813e30ae,43013,1731715110098, state=OPENING 2024-11-15T23:58:33,822 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T23:58:33,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:33,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T23:58:33,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:58:33,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:58:33,827 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T23:58:33,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:58:34,007 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T23:58:34,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52163, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T23:58:34,026 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T23:58:34,027 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T23:58:34,027 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T23:58:34,035 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0801813e30ae%2C43013%2C1731715110098.meta, suffix=.meta, logDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098, archiveDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/oldWALs, maxLogs=32 2024-11-15T23:58:34,059 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098/0801813e30ae%2C43013%2C1731715110098.meta.1731715114037.meta, exclude list is [], retry=0 2024-11-15T23:58:34,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43069,DS-3ff48f02-c27b-4ddf-8fcb-d3dce79566bf,DISK] 2024-11-15T23:58:34,070 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/WALs/0801813e30ae,43013,1731715110098/0801813e30ae%2C43013%2C1731715110098.meta.1731715114037.meta 2024-11-15T23:58:34,073 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35573:35573)] 2024-11-15T23:58:34,074 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:58:34,077 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T23:58:34,082 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T23:58:34,092 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T23:58:34,111 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T23:58:34,112 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:34,112 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T23:58:34,113 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T23:58:34,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T23:58:34,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T23:58:34,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:34,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:34,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T23:58:34,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T23:58:34,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:34,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:34,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T23:58:34,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T23:58:34,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:34,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:34,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T23:58:34,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T23:58:34,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:34,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T23:58:34,137 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T23:58:34,139 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740 2024-11-15T23:58:34,143 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740 2024-11-15T23:58:34,146 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T23:58:34,146 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T23:58:34,147 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:58:34,151 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T23:58:34,154 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68701354, jitterRate=0.023729950189590454}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:58:34,154 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T23:58:34,157 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731715114113Writing region info on filesystem at 1731715114114 (+1 ms)Initializing all the Stores at 1731715114116 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715114116Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715114117 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715114117Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731715114117Cleaning up temporary data from old regions at 1731715114146 (+29 ms)Running coprocessor post-open hooks at 1731715114155 (+9 ms)Region opened successfully at 1731715114157 (+2 ms) 2024-11-15T23:58:34,173 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731715113997 2024-11-15T23:58:34,198 DEBUG [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T23:58:34,199 INFO [RS_OPEN_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T23:58:34,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:58:34,203 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0801813e30ae,43013,1731715110098, state=OPEN 2024-11-15T23:58:34,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T23:58:34,214 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:34,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T23:58:34,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:58:34,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T23:58:34,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T23:58:34,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0801813e30ae,43013,1731715110098 in 387 msec 2024-11-15T23:58:34,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T23:58:34,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-15T23:58:34,236 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T23:58:34,236 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T23:58:34,271 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:34,273 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:34,303 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:34,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35393, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:34,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7550 sec 2024-11-15T23:58:34,345 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731715114345, completionTime=-1 2024-11-15T23:58:34,347 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T23:58:34,348 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T23:58:34,378 INFO [master/0801813e30ae:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T23:58:34,379 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731715174379 2024-11-15T23:58:34,379 INFO [master/0801813e30ae:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731715234379 2024-11-15T23:58:34,379 INFO [master/0801813e30ae:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-15T23:58:34,382 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,382 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,382 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,385 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0801813e30ae:38967, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,386 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,386 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,394 DEBUG [master/0801813e30ae:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T23:58:34,423 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.129sec 2024-11-15T23:58:34,425 INFO [master/0801813e30ae:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T23:58:34,426 INFO [master/0801813e30ae:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T23:58:34,427 INFO [master/0801813e30ae:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T23:58:34,428 INFO [master/0801813e30ae:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T23:58:34,428 INFO [master/0801813e30ae:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T23:58:34,429 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T23:58:34,430 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T23:58:34,444 DEBUG [master/0801813e30ae:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T23:58:34,445 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T23:58:34,446 INFO [master/0801813e30ae:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0801813e30ae,38967,1731715109050-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T23:58:34,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bed8dc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:34,456 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T23:58:34,456 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T23:58:34,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:34,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:34,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0d6658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:34,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:34,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d23fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:34,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:34,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:34,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:34,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:34,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0801813e30ae,38967,1731715109050 2024-11-15T23:58:34,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=6431 2024-11-15T23:58:34,611 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:58:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-15T23:58:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@603c91f2 2024-11-15T23:58:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:58:34,626 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:58:34,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:58:34,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:58:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:58:34,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:58:34,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T23:58:34,649 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:34,652 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:58:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:58:34,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741835_1011 (size=959) 2024-11-15T23:58:34,669 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-15T23:58:34,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741836_1012 (size=53) 2024-11-15T23:58:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:58:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:58:35,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:35,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 131fcc9fbcad104b408318bd0c6e140c, disabling compactions & flushes 2024-11-15T23:58:35,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. after waiting 0 ms 2024-11-15T23:58:35,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 131fcc9fbcad104b408318bd0c6e140c: Waiting for close lock at 1731715115080Disabling compacts and flushes for region at 1731715115080Disabling writes for close at 1731715115080Writing region close event to WAL at 1731715115080Closed at 1731715115080 2024-11-15T23:58:35,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:58:35,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715115083"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715115083"}]},"ts":"1731715115083"} 2024-11-15T23:58:35,099 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:58:35,101 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:58:35,105 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715115101"}]},"ts":"1731715115101"} 2024-11-15T23:58:35,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:58:35,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, ASSIGN}] 2024-11-15T23:58:35,117 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, ASSIGN 2024-11-15T23:58:35,119 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-15T23:58:35,272 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=131fcc9fbcad104b408318bd0c6e140c, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:58:35,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, ASSIGN because future has completed 2024-11-15T23:58:35,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:58:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:58:35,444 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,444 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:58:35,445 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,445 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:58:35,445 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,445 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,448 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,451 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 131fcc9fbcad104b408318bd0c6e140c columnFamilyName A 2024-11-15T23:58:35,451 DEBUG [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:35,453 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(327): Store=131fcc9fbcad104b408318bd0c6e140c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:35,453 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,456 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 131fcc9fbcad104b408318bd0c6e140c columnFamilyName B 2024-11-15T23:58:35,456 DEBUG [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:35,457 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(327): Store=131fcc9fbcad104b408318bd0c6e140c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:35,457 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,459 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 131fcc9fbcad104b408318bd0c6e140c columnFamilyName C 2024-11-15T23:58:35,460 DEBUG [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:58:35,460 INFO [StoreOpener-131fcc9fbcad104b408318bd0c6e140c-1 {}] regionserver.HStore(327): Store=131fcc9fbcad104b408318bd0c6e140c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:58:35,461 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,462 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,463 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,466 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,466 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,472 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:58:35,476 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,481 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:58:35,482 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 131fcc9fbcad104b408318bd0c6e140c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68341234, jitterRate=0.018363744020462036}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:58:35,482 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:35,484 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 131fcc9fbcad104b408318bd0c6e140c: Running coprocessor pre-open hook at 1731715115446Writing region info on filesystem at 1731715115446Initializing all the Stores at 1731715115448 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715115448Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715115448Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715115448Cleaning up temporary data from old regions at 1731715115466 (+18 ms)Running coprocessor post-open hooks at 1731715115482 (+16 ms)Region opened successfully at 1731715115484 (+2 ms) 2024-11-15T23:58:35,487 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., pid=6, masterSystemTime=1731715115435 2024-11-15T23:58:35,492 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,492 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:35,494 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=131fcc9fbcad104b408318bd0c6e140c, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:58:35,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:58:35,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T23:58:35,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 in 225 msec 2024-11-15T23:58:35,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T23:58:35,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, ASSIGN in 398 msec 2024-11-15T23:58:35,520 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:58:35,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715115520"}]},"ts":"1731715115520"} 2024-11-15T23:58:35,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:58:35,527 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:58:35,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 890 msec 2024-11-15T23:58:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T23:58:35,791 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:35,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a13e638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a99da4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e1d172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:58:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-15T23:58:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5effb952 2024-11-15T23:58:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:58:35,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:58:35,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0774b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f897912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3274ad1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,840 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b147913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408cf13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da6ed4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43659da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745237a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ba2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c6d95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592e422b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69964181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd942f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d03237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24472a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2335953a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552dc5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e684afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b3e7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288fb03e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c75040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65766813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:58:35,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:58:35,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:58:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:58:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:58:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:58:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:58:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:35,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:58:35,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aacb006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:58:35,899 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,899 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,899 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,899 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,902 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,904 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,905 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,911 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,931 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,965 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:35,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:35,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:35,994 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:58:35,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:58:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:58:36,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:58:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T23:58:36,010 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:58:36,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:36,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:36,075 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:36,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:58:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:36,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:58:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:36,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/b3773bd8fc0c418ea99bcfde9ae8a48e is 50, key is test_row_0/A:col10/1731715116048/Put/seqid=0 2024-11-15T23:58:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741837_1013 (size=11997) 2024-11-15T23:58:36,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/b3773bd8fc0c418ea99bcfde9ae8a48e 2024-11-15T23:58:36,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:58:36,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715176436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715176419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715176419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715176448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715176451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dd19992bfc464778a2b0d8acb320a736 is 50, key is test_row_0/B:col10/1731715116048/Put/seqid=0 2024-11-15T23:58:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741838_1014 (size=9657) 2024-11-15T23:58:36,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dd19992bfc464778a2b0d8acb320a736 2024-11-15T23:58:36,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:58:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:36,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/5410620719b44c9d9509c3d8405eedca is 50, key is test_row_0/C:col10/1731715116048/Put/seqid=0 2024-11-15T23:58:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715176621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715176621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741839_1015 (size=9657) 2024-11-15T23:58:36,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715176624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715176624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/5410620719b44c9d9509c3d8405eedca 2024-11-15T23:58:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715176622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/b3773bd8fc0c418ea99bcfde9ae8a48e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e 2024-11-15T23:58:36,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T23:58:36,687 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T23:58:36,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dd19992bfc464778a2b0d8acb320a736 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736 2024-11-15T23:58:36,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736, entries=100, sequenceid=13, filesize=9.4 K 2024-11-15T23:58:36,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/5410620719b44c9d9509c3d8405eedca as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca 2024-11-15T23:58:36,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca, entries=100, sequenceid=13, filesize=9.4 K 2024-11-15T23:58:36,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 131fcc9fbcad104b408318bd0c6e140c in 666ms, sequenceid=13, compaction requested=false 2024-11-15T23:58:36,764 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T23:58:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:36,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T23:58:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:36,784 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:58:36,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fada121d5c5a4d568baabe4147aadf8d is 50, key is test_row_0/A:col10/1731715116403/Put/seqid=0 2024-11-15T23:58:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:36,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741840_1016 (size=12001) 2024-11-15T23:58:36,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715176903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715176906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715176907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715176912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:36,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715176918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715177031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715177031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715177034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715177039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715177043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:37,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715177241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715177241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715177251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715177253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,261 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fada121d5c5a4d568baabe4147aadf8d 2024-11-15T23:58:37,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715177261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/da4a21fc02a041378e9237d749698898 is 50, key is test_row_0/B:col10/1731715116403/Put/seqid=0 2024-11-15T23:58:37,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741841_1017 (size=12001) 2024-11-15T23:58:37,322 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/da4a21fc02a041378e9237d749698898 2024-11-15T23:58:37,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/d8f67fd57d154c19a3f5596ae2bbf069 is 50, key is test_row_0/C:col10/1731715116403/Put/seqid=0 2024-11-15T23:58:37,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741842_1018 (size=12001) 2024-11-15T23:58:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715177561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715177562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715177563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715177563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715177580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:37,804 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/d8f67fd57d154c19a3f5596ae2bbf069 2024-11-15T23:58:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fada121d5c5a4d568baabe4147aadf8d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d 2024-11-15T23:58:37,833 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T23:58:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/da4a21fc02a041378e9237d749698898 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898 2024-11-15T23:58:37,851 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T23:58:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/d8f67fd57d154c19a3f5596ae2bbf069 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069 2024-11-15T23:58:37,866 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T23:58:37,868 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 131fcc9fbcad104b408318bd0c6e140c in 1083ms, sequenceid=37, compaction requested=false 2024-11-15T23:58:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T23:58:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T23:58:37,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T23:58:37,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-15T23:58:37,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8760 sec 2024-11-15T23:58:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:38,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:58:38,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/9b5610bf0b954856ac739c97e3c72a9f is 50, key is test_row_0/A:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T23:58:38,171 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:38,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741843_1019 (size=12001) 2024-11-15T23:58:38,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T23:58:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:38,177 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:38,179 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:38,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:38,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715178190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715178193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715178193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715178200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715178203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:38,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715178310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715178310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715178321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715178321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715178321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:38,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:38,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:38,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715178530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715178540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715178540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715178541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715178554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:38,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:38,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/9b5610bf0b954856ac739c97e3c72a9f 2024-11-15T23:58:38,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/94aa5c144e4f4313a880dba32191e4c9 is 50, key is test_row_0/B:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:38,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741844_1020 (size=12001) 2024-11-15T23:58:38,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/94aa5c144e4f4313a880dba32191e4c9 2024-11-15T23:58:38,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/8772df749a514361b8193c490b38efdb is 50, key is test_row_0/C:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:38,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741845_1021 (size=12001) 2024-11-15T23:58:38,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/8772df749a514361b8193c490b38efdb 2024-11-15T23:58:38,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/9b5610bf0b954856ac739c97e3c72a9f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f 2024-11-15T23:58:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:38,805 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T23:58:38,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:38,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T23:58:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/94aa5c144e4f4313a880dba32191e4c9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9 2024-11-15T23:58:38,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T23:58:38,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/8772df749a514361b8193c490b38efdb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb 2024-11-15T23:58:38,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T23:58:38,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 131fcc9fbcad104b408318bd0c6e140c in 777ms, sequenceid=52, compaction requested=true 2024-11-15T23:58:38,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:38,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:38,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:38,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:38,938 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:38,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:38,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:38,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:58:38,942 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T23:58:38,948 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:38,965 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:38,965 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:38,968 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:38,968 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,969 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=32.9 K 2024-11-15T23:58:38,970 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:38,970 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:38,970 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=35.2 K 2024-11-15T23:58:38,971 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting dd19992bfc464778a2b0d8acb320a736, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731715116048 2024-11-15T23:58:38,972 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting da4a21fc02a041378e9237d749698898, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715116403 2024-11-15T23:58:38,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/3085dd39643242c6a604008c7d4fea16 is 50, key is test_row_0/A:col10/1731715118879/Put/seqid=0 2024-11-15T23:58:38,977 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3773bd8fc0c418ea99bcfde9ae8a48e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731715116048 2024-11-15T23:58:38,980 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting fada121d5c5a4d568baabe4147aadf8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715116403 2024-11-15T23:58:38,981 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5610bf0b954856ac739c97e3c72a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:38,982 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 94aa5c144e4f4313a880dba32191e4c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:38,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715178964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715178968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715178976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715178977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715178979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741846_1022 (size=14341) 2024-11-15T23:58:39,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/3085dd39643242c6a604008c7d4fea16 2024-11-15T23:58:39,047 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:39,053 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#11 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:39,055 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e37e6bf1d639490083580e70e2bb902f is 50, key is test_row_0/B:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:39,059 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/9ca110e2c88f47f49c067e1e438e77bd is 50, key is test_row_0/A:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:39,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/21e2be35e8994021a4793cd374c215ab is 50, key is test_row_0/B:col10/1731715118879/Put/seqid=0 2024-11-15T23:58:39,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741847_1023 (size=12104) 2024-11-15T23:58:39,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741849_1025 (size=12104) 2024-11-15T23:58:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715179111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715179111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715179111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741848_1024 (size=12001) 2024-11-15T23:58:39,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/21e2be35e8994021a4793cd374c215ab 2024-11-15T23:58:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715179120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,153 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/9ca110e2c88f47f49c067e1e438e77bd as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9ca110e2c88f47f49c067e1e438e77bd 2024-11-15T23:58:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715179121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/f757a4f6c12a45bcba9f4308e1e172cb is 50, key is test_row_0/C:col10/1731715118879/Put/seqid=0 2024-11-15T23:58:39,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,183 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 9ca110e2c88f47f49c067e1e438e77bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:39,183 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:39,186 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715118872; duration=0sec 2024-11-15T23:58:39,187 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:39,187 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:39,187 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:39,194 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:39,194 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:39,194 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,194 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=32.9 K 2024-11-15T23:58:39,199 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5410620719b44c9d9509c3d8405eedca, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731715116048 2024-11-15T23:58:39,200 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f67fd57d154c19a3f5596ae2bbf069, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715116403 2024-11-15T23:58:39,202 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8772df749a514361b8193c490b38efdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:39,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741850_1026 (size=12001) 2024-11-15T23:58:39,256 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#14 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:39,257 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/2efb554c503d482d8fd937fcd10f3ef2 is 50, key is test_row_0/C:col10/1731715118084/Put/seqid=0 2024-11-15T23:58:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:39,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741851_1027 (size=12104) 2024-11-15T23:58:39,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715179352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715179352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715179352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715179361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715179361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,370 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/2efb554c503d482d8fd937fcd10f3ef2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2efb554c503d482d8fd937fcd10f3ef2 2024-11-15T23:58:39,434 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 2efb554c503d482d8fd937fcd10f3ef2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:39,434 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:39,434 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715118938; duration=0sec 2024-11-15T23:58:39,435 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:39,435 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:39,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,529 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e37e6bf1d639490083580e70e2bb902f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e37e6bf1d639490083580e70e2bb902f 2024-11-15T23:58:39,549 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into e37e6bf1d639490083580e70e2bb902f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:39,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:39,549 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715118938; duration=0sec 2024-11-15T23:58:39,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:39,550 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:39,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/f757a4f6c12a45bcba9f4308e1e172cb 2024-11-15T23:58:39,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/3085dd39643242c6a604008c7d4fea16 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16 2024-11-15T23:58:39,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16, entries=200, sequenceid=78, filesize=14.0 K 2024-11-15T23:58:39,646 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:58:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/21e2be35e8994021a4793cd374c215ab as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab 2024-11-15T23:58:39,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T23:58:39,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/f757a4f6c12a45bcba9f4308e1e172cb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb 2024-11-15T23:58:39,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715179671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715179671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715179672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715179672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:39,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T23:58:39,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 131fcc9fbcad104b408318bd0c6e140c in 755ms, sequenceid=78, compaction requested=false 2024-11-15T23:58:39,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:39,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:58:39,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/dc71f39482eb4fa6a2c03828594be56f is 50, key is test_row_0/A:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741852_1028 (size=14341) 2024-11-15T23:58:39,788 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T23:58:39,788 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T23:58:39,801 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T23:58:39,801 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T23:58:39,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T23:58:39,802 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T23:58:39,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T23:58:39,803 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T23:58:39,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:39,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:39,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715180005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715180130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/dc71f39482eb4fa6a2c03828594be56f 2024-11-15T23:58:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715180180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3c4162519ef043db9fc590b5ab497806 is 50, key is test_row_0/B:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:40,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715180210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715180210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715180210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741853_1029 (size=12001) 2024-11-15T23:58:40,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715180342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3c4162519ef043db9fc590b5ab497806 2024-11-15T23:58:40,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715180661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:40,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/7afb887114dd47e1ba0a794ed8d7bbbc is 50, key is test_row_0/C:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741854_1030 (size=12001) 2024-11-15T23:58:40,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:40,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/7afb887114dd47e1ba0a794ed8d7bbbc 2024-11-15T23:58:41,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/dc71f39482eb4fa6a2c03828594be56f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f 2024-11-15T23:58:41,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f, entries=200, sequenceid=92, filesize=14.0 K 2024-11-15T23:58:41,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3c4162519ef043db9fc590b5ab497806 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806 2024-11-15T23:58:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T23:58:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/7afb887114dd47e1ba0a794ed8d7bbbc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc 2024-11-15T23:58:41,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T23:58:41,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 131fcc9fbcad104b408318bd0c6e140c in 1482ms, sequenceid=92, compaction requested=true 2024-11-15T23:58:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:41,186 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:41,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:41,189 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:41,189 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:41,189 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,189 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9ca110e2c88f47f49c067e1e438e77bd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=39.8 K 2024-11-15T23:58:41,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:41,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:41,190 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,190 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e37e6bf1d639490083580e70e2bb902f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=35.3 K 2024-11-15T23:58:41,192 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e37e6bf1d639490083580e70e2bb902f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:41,192 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca110e2c88f47f49c067e1e438e77bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:41,193 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e2be35e8994021a4793cd374c215ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731715118196 2024-11-15T23:58:41,193 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3085dd39643242c6a604008c7d4fea16, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731715118196 2024-11-15T23:58:41,194 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4162519ef043db9fc590b5ab497806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118963 2024-11-15T23:58:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:41,194 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:41,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T23:58:41,194 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc71f39482eb4fa6a2c03828594be56f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118948 2024-11-15T23:58:41,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/44c0dc8912784e99b44fb37bd0d4af14 is 50, key is test_row_0/A:col10/1731715119985/Put/seqid=0 2024-11-15T23:58:41,222 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:41,223 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/4cdf5afa7a3f47c28399f30531a28d17 is 50, key is test_row_0/B:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:41,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715181228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715181230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715181231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715181236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,244 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715181238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,247 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,261 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:41,262 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/278b503d16ec49aaba8c12338cc3fcd8 is 50, key is test_row_0/A:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741855_1031 (size=14341) 2024-11-15T23:58:41,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/44c0dc8912784e99b44fb37bd0d4af14 2024-11-15T23:58:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741856_1032 (size=12207) 2024-11-15T23:58:41,336 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/4cdf5afa7a3f47c28399f30531a28d17 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4cdf5afa7a3f47c28399f30531a28d17 2024-11-15T23:58:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741857_1033 (size=12207) 2024-11-15T23:58:41,349 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into 4cdf5afa7a3f47c28399f30531a28d17(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:41,350 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:41,350 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715121186; duration=0sec 2024-11-15T23:58:41,350 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:41,350 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:41,351 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:41,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e5ac1eabe0894d468bb79485a0553ebc is 50, key is test_row_0/B:col10/1731715119985/Put/seqid=0 2024-11-15T23:58:41,353 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:41,353 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:41,353 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,353 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2efb554c503d482d8fd937fcd10f3ef2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=35.3 K 2024-11-15T23:58:41,354 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efb554c503d482d8fd937fcd10f3ef2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715118084 2024-11-15T23:58:41,355 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f757a4f6c12a45bcba9f4308e1e172cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731715118196 2024-11-15T23:58:41,355 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7afb887114dd47e1ba0a794ed8d7bbbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118963 2024-11-15T23:58:41,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715181350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715181361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715181361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,408 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:41,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741858_1034 (size=12001) 2024-11-15T23:58:41,418 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/28262b4f3ff34b3e9df61aad3d70f7ca is 50, key is test_row_0/C:col10/1731715119700/Put/seqid=0 2024-11-15T23:58:41,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741859_1035 (size=12207) 2024-11-15T23:58:41,535 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/28262b4f3ff34b3e9df61aad3d70f7ca as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/28262b4f3ff34b3e9df61aad3d70f7ca 2024-11-15T23:58:41,547 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 28262b4f3ff34b3e9df61aad3d70f7ca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:41,547 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:41,547 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715121187; duration=0sec 2024-11-15T23:58:41,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:41,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:41,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715181574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715181581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715181581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,759 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/278b503d16ec49aaba8c12338cc3fcd8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/278b503d16ec49aaba8c12338cc3fcd8 2024-11-15T23:58:41,776 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 278b503d16ec49aaba8c12338cc3fcd8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:41,776 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:41,776 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715121186; duration=0sec 2024-11-15T23:58:41,776 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:41,777 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:41,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e5ac1eabe0894d468bb79485a0553ebc 2024-11-15T23:58:41,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1452953f8fb47268882fb0348bbb6ab is 50, key is test_row_0/C:col10/1731715119985/Put/seqid=0 2024-11-15T23:58:41,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:41,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741860_1036 (size=12001) 2024-11-15T23:58:41,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1452953f8fb47268882fb0348bbb6ab 2024-11-15T23:58:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715181890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715181890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715181891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:41,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/44c0dc8912784e99b44fb37bd0d4af14 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14 2024-11-15T23:58:41,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14, entries=200, sequenceid=118, filesize=14.0 K 2024-11-15T23:58:41,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e5ac1eabe0894d468bb79485a0553ebc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc 2024-11-15T23:58:41,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T23:58:41,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1452953f8fb47268882fb0348bbb6ab as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab 2024-11-15T23:58:41,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T23:58:41,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 131fcc9fbcad104b408318bd0c6e140c in 788ms, sequenceid=118, compaction requested=false 2024-11-15T23:58:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:42,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T23:58:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,039 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T23:58:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/5170b29cd8454bde836fb77889dccb59 is 50, key is test_row_1/A:col10/1731715121233/Put/seqid=0 2024-11-15T23:58:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741861_1037 (size=9657) 2024-11-15T23:58:42,099 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/5170b29cd8454bde836fb77889dccb59 2024-11-15T23:58:42,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dedf52705f3f4269b2eb0e8f370b51e5 is 50, key is test_row_1/B:col10/1731715121233/Put/seqid=0 2024-11-15T23:58:42,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741862_1038 (size=9657) 2024-11-15T23:58:42,162 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dedf52705f3f4269b2eb0e8f370b51e5 2024-11-15T23:58:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/74aa8bea0c314831831718fb0b8db76c is 50, key is test_row_1/C:col10/1731715121233/Put/seqid=0 2024-11-15T23:58:42,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741863_1039 (size=9657) 2024-11-15T23:58:42,259 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/74aa8bea0c314831831718fb0b8db76c 2024-11-15T23:58:42,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/5170b29cd8454bde836fb77889dccb59 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59 2024-11-15T23:58:42,288 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59, entries=100, sequenceid=131, filesize=9.4 K 2024-11-15T23:58:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/dedf52705f3f4269b2eb0e8f370b51e5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5 2024-11-15T23:58:42,298 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5, entries=100, sequenceid=131, filesize=9.4 K 2024-11-15T23:58:42,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/74aa8bea0c314831831718fb0b8db76c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c 2024-11-15T23:58:42,316 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c, entries=100, sequenceid=131, filesize=9.4 K 2024-11-15T23:58:42,319 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 131fcc9fbcad104b408318bd0c6e140c in 280ms, sequenceid=131, compaction requested=true 2024-11-15T23:58:42,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:42,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T23:58:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T23:58:42,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T23:58:42,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1440 sec 2024-11-15T23:58:42,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 4.1550 sec 2024-11-15T23:58:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T23:58:42,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:42,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T23:58:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:42,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:42,346 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:42,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:42,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:42,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:58:42,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/e4b2e2a67b964ac59b167f0d9f1e4553 is 50, key is test_row_0/A:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:42,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741864_1040 (size=16931) 2024-11-15T23:58:42,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715182600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715182600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715182602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:42,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715182710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715182710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715182710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:42,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:42,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715182930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715182930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:42,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715182931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:42,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/e4b2e2a67b964ac59b167f0d9f1e4553 2024-11-15T23:58:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:42,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/a1b9ed2fe7e1411e81b2de22228d149e is 50, key is test_row_0/B:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:42,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:42,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741865_1041 (size=12151) 2024-11-15T23:58:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715183250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715183250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715183250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715183260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,262 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4291 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715183281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,283 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/a1b9ed2fe7e1411e81b2de22228d149e 2024-11-15T23:58:43,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e9763427841142ccb1cc7401b4b3e785 is 50, key is test_row_0/C:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741866_1042 (size=12151) 2024-11-15T23:58:43,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e9763427841142ccb1cc7401b4b3e785 2024-11-15T23:58:43,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/e4b2e2a67b964ac59b167f0d9f1e4553 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553 2024-11-15T23:58:43,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553, entries=250, sequenceid=144, filesize=16.5 K 2024-11-15T23:58:43,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/a1b9ed2fe7e1411e81b2de22228d149e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e 2024-11-15T23:58:43,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e, entries=150, sequenceid=144, filesize=11.9 K 2024-11-15T23:58:43,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e9763427841142ccb1cc7401b4b3e785 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785 2024-11-15T23:58:43,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785, entries=150, sequenceid=144, filesize=11.9 K 2024-11-15T23:58:43,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 131fcc9fbcad104b408318bd0c6e140c in 1111ms, sequenceid=144, compaction requested=true 2024-11-15T23:58:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:43,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:43,580 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:58:43,581 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:43,582 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:43,582 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:43,583 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,583 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/278b503d16ec49aaba8c12338cc3fcd8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=51.9 K 2024-11-15T23:58:43,584 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 278b503d16ec49aaba8c12338cc3fcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118963 2024-11-15T23:58:43,585 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c0dc8912784e99b44fb37bd0d4af14, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731715119905 2024-11-15T23:58:43,585 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:43,585 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:43,586 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,586 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4cdf5afa7a3f47c28399f30531a28d17, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=44.9 K 2024-11-15T23:58:43,586 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdf5afa7a3f47c28399f30531a28d17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118963 2024-11-15T23:58:43,588 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ac1eabe0894d468bb79485a0553ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731715119985 2024-11-15T23:58:43,589 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5170b29cd8454bde836fb77889dccb59, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731715121233 2024-11-15T23:58:43,589 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting dedf52705f3f4269b2eb0e8f370b51e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731715121233 2024-11-15T23:58:43,589 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b2e2a67b964ac59b167f0d9f1e4553, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:43,590 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b9ed2fe7e1411e81b2de22228d149e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:43,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T23:58:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,612 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T23:58:43,630 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:43,631 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e73acb3825b342289051beac7336cbc3 is 50, key is test_row_0/B:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/345bbdfca3c949e7b71612a0daaa03a0 is 50, key is test_row_0/A:col10/1731715122599/Put/seqid=0 2024-11-15T23:58:43,640 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#31 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:43,641 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/36972fbeae014a26bf9e13684ed33bdf is 50, key is test_row_0/A:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:43,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741868_1044 (size=12151) 2024-11-15T23:58:43,656 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/345bbdfca3c949e7b71612a0daaa03a0 2024-11-15T23:58:43,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741867_1043 (size=12493) 2024-11-15T23:58:43,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/e73acb3825b342289051beac7336cbc3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e73acb3825b342289051beac7336cbc3 2024-11-15T23:58:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741869_1045 (size=12493) 2024-11-15T23:58:43,690 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into e73acb3825b342289051beac7336cbc3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:43,690 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:43,691 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=12, startTime=1731715123580; duration=0sec 2024-11-15T23:58:43,691 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:43,691 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:43,691 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:43,694 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/36972fbeae014a26bf9e13684ed33bdf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/36972fbeae014a26bf9e13684ed33bdf 2024-11-15T23:58:43,695 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:43,695 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:43,695 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:43,695 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/28262b4f3ff34b3e9df61aad3d70f7ca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=44.9 K 2024-11-15T23:58:43,697 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28262b4f3ff34b3e9df61aad3d70f7ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715118963 2024-11-15T23:58:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/310185182ade4702a92802a5c130a1d9 is 50, key is test_row_0/B:col10/1731715122599/Put/seqid=0 2024-11-15T23:58:43,698 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1452953f8fb47268882fb0348bbb6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731715119985 2024-11-15T23:58:43,700 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74aa8bea0c314831831718fb0b8db76c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731715121233 2024-11-15T23:58:43,700 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9763427841142ccb1cc7401b4b3e785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:43,707 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 36972fbeae014a26bf9e13684ed33bdf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:43,707 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:43,707 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=12, startTime=1731715123579; duration=0sec 2024-11-15T23:58:43,707 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:43,707 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:43,724 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:43,725 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/200c9f165773484392e10beee620d24d is 50, key is test_row_0/C:col10/1731715122417/Put/seqid=0 2024-11-15T23:58:43,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741870_1046 (size=12151) 2024-11-15T23:58:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/310185182ade4702a92802a5c130a1d9 2024-11-15T23:58:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e04d5d5d00d749f0bebc0e4dda30b771 is 50, key is test_row_0/C:col10/1731715122599/Put/seqid=0 2024-11-15T23:58:43,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741871_1047 (size=12493) 2024-11-15T23:58:43,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741872_1048 (size=12151) 2024-11-15T23:58:43,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715183806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715183814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715183819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,826 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/200c9f165773484392e10beee620d24d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/200c9f165773484392e10beee620d24d 2024-11-15T23:58:43,838 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 200c9f165773484392e10beee620d24d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:43,839 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:43,839 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=12, startTime=1731715123580; duration=0sec 2024-11-15T23:58:43,839 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:43,839 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:43,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715183931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715183932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715183934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715184142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715184142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715184145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,198 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e04d5d5d00d749f0bebc0e4dda30b771 2024-11-15T23:58:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/345bbdfca3c949e7b71612a0daaa03a0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0 2024-11-15T23:58:44,234 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T23:58:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/310185182ade4702a92802a5c130a1d9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9 2024-11-15T23:58:44,271 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T23:58:44,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e04d5d5d00d749f0bebc0e4dda30b771 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771 2024-11-15T23:58:44,287 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T23:58:44,290 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 131fcc9fbcad104b408318bd0c6e140c in 677ms, sequenceid=167, compaction requested=false 2024-11-15T23:58:44,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:44,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:44,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T23:58:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T23:58:44,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T23:58:44,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-11-15T23:58:44,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9560 sec 2024-11-15T23:58:44,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:44,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:58:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:44,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f49b269a50f34204b74542f6edd63ccf is 50, key is test_row_0/A:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T23:58:44,502 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:44,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T23:58:44,506 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:44,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:44,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:44,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741873_1049 (size=14541) 2024-11-15T23:58:44,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715184575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715184575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715184577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:44,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:44,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715184690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715184691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715184691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:44,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:44,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:44,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715184900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715184910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715184912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:44,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f49b269a50f34204b74542f6edd63ccf 2024-11-15T23:58:44,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3fd1c804356c4782abe4db51912328aa is 50, key is test_row_0/B:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:44,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741874_1050 (size=12151) 2024-11-15T23:58:45,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:45,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715185210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715185220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715185220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3fd1c804356c4782abe4db51912328aa 2024-11-15T23:58:45,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/662fbe908ccf420491b2da0cdad22259 is 50, key is test_row_0/C:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:45,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741875_1051 (size=12151) 2024-11-15T23:58:45,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:45,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715185720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715185729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715185730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:45,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:45,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/662fbe908ccf420491b2da0cdad22259 2024-11-15T23:58:45,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f49b269a50f34204b74542f6edd63ccf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf 2024-11-15T23:58:45,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf, entries=200, sequenceid=184, filesize=14.2 K 2024-11-15T23:58:45,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/3fd1c804356c4782abe4db51912328aa as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa 2024-11-15T23:58:45,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T23:58:45,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/662fbe908ccf420491b2da0cdad22259 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259 2024-11-15T23:58:45,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T23:58:45,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 131fcc9fbcad104b408318bd0c6e140c in 1483ms, sequenceid=184, compaction requested=true 2024-11-15T23:58:45,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:45,951 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:45,952 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:45,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:45,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:45,954 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:45,955 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:45,955 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:45,955 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,955 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:45,955 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/36972fbeae014a26bf9e13684ed33bdf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=38.3 K 2024-11-15T23:58:45,955 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,955 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e73acb3825b342289051beac7336cbc3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=35.9 K 2024-11-15T23:58:45,957 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e73acb3825b342289051beac7336cbc3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:45,957 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36972fbeae014a26bf9e13684ed33bdf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:45,958 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 310185182ade4702a92802a5c130a1d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731715122582 2024-11-15T23:58:45,958 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345bbdfca3c949e7b71612a0daaa03a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731715122582 2024-11-15T23:58:45,959 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd1c804356c4782abe4db51912328aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:45,959 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f49b269a50f34204b74542f6edd63ccf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:45,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T23:58:45,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:45,976 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T23:58:46,000 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:46,001 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/b4a973e3bf2e47c98525dde24fa6bad3 is 50, key is test_row_0/B:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:46,009 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:46,009 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/d6cf6201424c4a9e908975423cd6e18f is 50, key is test_row_0/A:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/356f0eba56394e43871be5d463fc625d is 50, key is test_row_0/A:col10/1731715124575/Put/seqid=0 2024-11-15T23:58:46,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741876_1052 (size=12595) 2024-11-15T23:58:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741877_1053 (size=12151) 2024-11-15T23:58:46,079 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/356f0eba56394e43871be5d463fc625d 2024-11-15T23:58:46,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741878_1054 (size=12595) 2024-11-15T23:58:46,106 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/b4a973e3bf2e47c98525dde24fa6bad3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/b4a973e3bf2e47c98525dde24fa6bad3 2024-11-15T23:58:46,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/d6cf6201424c4a9e908975423cd6e18f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d6cf6201424c4a9e908975423cd6e18f 2024-11-15T23:58:46,138 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into b4a973e3bf2e47c98525dde24fa6bad3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:46,138 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:46,138 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715125952; duration=0sec 2024-11-15T23:58:46,139 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:46,139 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:46,139 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:46,141 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into d6cf6201424c4a9e908975423cd6e18f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:46,141 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:46,141 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715125951; duration=0sec 2024-11-15T23:58:46,142 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:46,142 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:46,142 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:46,142 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:46,143 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:46,143 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/200c9f165773484392e10beee620d24d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=35.9 K 2024-11-15T23:58:46,143 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 200c9f165773484392e10beee620d24d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731715122417 2024-11-15T23:58:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/5839d16b8fdb487ba5e5152d33e64163 is 50, key is test_row_0/B:col10/1731715124575/Put/seqid=0 2024-11-15T23:58:46,144 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e04d5d5d00d749f0bebc0e4dda30b771, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731715122582 2024-11-15T23:58:46,148 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 662fbe908ccf420491b2da0cdad22259, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:46,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741879_1055 (size=12151) 2024-11-15T23:58:46,186 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#43 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:46,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/844fd459a0f84954aaa625a2a8cf8463 is 50, key is test_row_0/C:col10/1731715123800/Put/seqid=0 2024-11-15T23:58:46,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741880_1056 (size=12595) 2024-11-15T23:58:46,270 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/844fd459a0f84954aaa625a2a8cf8463 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/844fd459a0f84954aaa625a2a8cf8463 2024-11-15T23:58:46,284 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 844fd459a0f84954aaa625a2a8cf8463(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:46,284 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:46,284 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715125953; duration=0sec 2024-11-15T23:58:46,284 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:46,284 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:46,582 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/5839d16b8fdb487ba5e5152d33e64163 2024-11-15T23:58:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/15f15f7b677e43df83a06b905b7ed2a3 is 50, key is test_row_0/C:col10/1731715124575/Put/seqid=0 2024-11-15T23:58:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:46,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741881_1057 (size=12151) 2024-11-15T23:58:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:46,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715186780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715186779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715186782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715186890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:46,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715186890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715186900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715187100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,107 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/15f15f7b677e43df83a06b905b7ed2a3 2024-11-15T23:58:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715187111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/356f0eba56394e43871be5d463fc625d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d 2024-11-15T23:58:47,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715187121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,141 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T23:58:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/5839d16b8fdb487ba5e5152d33e64163 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163 2024-11-15T23:58:47,151 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T23:58:47,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/15f15f7b677e43df83a06b905b7ed2a3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3 2024-11-15T23:58:47,171 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T23:58:47,173 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 131fcc9fbcad104b408318bd0c6e140c in 1197ms, sequenceid=206, compaction requested=false 2024-11-15T23:58:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:47,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T23:58:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T23:58:47,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T23:58:47,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6680 sec 2024-11-15T23:58:47,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.6800 sec 2024-11-15T23:58:47,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:47,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T23:58:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:47,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/d0d4dc08682a460399691d00c0893939 is 50, key is test_row_0/A:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:47,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741882_1058 (size=14541) 2024-11-15T23:58:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715187410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715187410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715187419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715187421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715187430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715187530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715187530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715187741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715187741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/d0d4dc08682a460399691d00c0893939 2024-11-15T23:58:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/defb180cd76948cb892bf997c3c8c937 is 50, key is test_row_0/B:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:47,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741883_1059 (size=12151) 2024-11-15T23:58:47,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715187930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715187940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715187950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715188062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715188073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/defb180cd76948cb892bf997c3c8c937 2024-11-15T23:58:48,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/ecbfbcf72d4d4e39bf7e011ece92a72e is 50, key is test_row_0/C:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:48,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741884_1060 (size=12151) 2024-11-15T23:58:48,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/ecbfbcf72d4d4e39bf7e011ece92a72e 2024-11-15T23:58:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/d0d4dc08682a460399691d00c0893939 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939 2024-11-15T23:58:48,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939, entries=200, sequenceid=224, filesize=14.2 K 2024-11-15T23:58:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/defb180cd76948cb892bf997c3c8c937 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937 2024-11-15T23:58:48,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T23:58:48,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/ecbfbcf72d4d4e39bf7e011ece92a72e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e 2024-11-15T23:58:48,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T23:58:48,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 131fcc9fbcad104b408318bd0c6e140c in 1125ms, sequenceid=224, compaction requested=true 2024-11-15T23:58:48,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:48,416 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:48,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:58:48,418 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:48,421 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:48,421 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:48,421 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,422 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d6cf6201424c4a9e908975423cd6e18f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=38.4 K 2024-11-15T23:58:48,423 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cf6201424c4a9e908975423cd6e18f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:48,425 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356f0eba56394e43871be5d463fc625d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731715124558 2024-11-15T23:58:48,426 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d4dc08682a460399691d00c0893939, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:48,427 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:48,427 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:48,427 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,427 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/b4a973e3bf2e47c98525dde24fa6bad3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.0 K 2024-11-15T23:58:48,433 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b4a973e3bf2e47c98525dde24fa6bad3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:48,437 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5839d16b8fdb487ba5e5152d33e64163, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731715124558 2024-11-15T23:58:48,441 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting defb180cd76948cb892bf997c3c8c937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:48,444 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:48,445 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/25aadd1c0f644f059e4a4eef0aceddd8 is 50, key is test_row_0/A:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:48,472 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:48,473 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/4f914197c4874aab89be6d0fb6de0a0a is 50, key is test_row_0/B:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:48,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741885_1061 (size=12697) 2024-11-15T23:58:48,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741886_1062 (size=12697) 2024-11-15T23:58:48,526 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/25aadd1c0f644f059e4a4eef0aceddd8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/25aadd1c0f644f059e4a4eef0aceddd8 2024-11-15T23:58:48,536 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 25aadd1c0f644f059e4a4eef0aceddd8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:48,536 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:48,536 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715128416; duration=0sec 2024-11-15T23:58:48,536 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:48,536 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:48,536 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:48,540 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/4f914197c4874aab89be6d0fb6de0a0a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4f914197c4874aab89be6d0fb6de0a0a 2024-11-15T23:58:48,541 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:48,541 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:48,541 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,542 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/844fd459a0f84954aaa625a2a8cf8463, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.0 K 2024-11-15T23:58:48,542 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844fd459a0f84954aaa625a2a8cf8463, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731715123800 2024-11-15T23:58:48,542 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f15f7b677e43df83a06b905b7ed2a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731715124558 2024-11-15T23:58:48,543 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbfbcf72d4d4e39bf7e011ece92a72e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:48,549 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into 4f914197c4874aab89be6d0fb6de0a0a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:48,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:48,549 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715128418; duration=0sec 2024-11-15T23:58:48,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:48,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:48,555 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:48,556 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e0041478d8ea4a49855087cfc4a5eada is 50, key is test_row_0/C:col10/1731715127285/Put/seqid=0 2024-11-15T23:58:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:48,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:48,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T23:58:48,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741887_1063 (size=12697) 2024-11-15T23:58:48,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/62e19362358348a8a56ffe58bdf63c69 is 50, key is test_row_0/A:col10/1731715128585/Put/seqid=0 2024-11-15T23:58:48,626 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/e0041478d8ea4a49855087cfc4a5eada as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e0041478d8ea4a49855087cfc4a5eada 2024-11-15T23:58:48,635 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into e0041478d8ea4a49855087cfc4a5eada(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:48,635 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:48,635 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715128418; duration=0sec 2024-11-15T23:58:48,635 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:48,635 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:48,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715188647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741888_1064 (size=14541) 2024-11-15T23:58:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715188654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T23:58:48,674 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:48,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T23:58:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:48,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:48,681 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:48,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:48,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715188760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715188770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:48,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:48,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:48,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:48,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:48,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715188950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715188950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,954 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,956 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715188960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,961 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715188980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:48,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715188986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:48,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:49,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/62e19362358348a8a56ffe58bdf63c69 2024-11-15T23:58:49,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0cd11e9c2e6e42b086e5690172829c3f is 50, key is test_row_0/B:col10/1731715128585/Put/seqid=0 2024-11-15T23:58:49,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741889_1065 (size=12151) 2024-11-15T23:58:49,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715189293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715189300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:49,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0cd11e9c2e6e42b086e5690172829c3f 2024-11-15T23:58:49,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 is 50, key is test_row_0/C:col10/1731715128585/Put/seqid=0 2024-11-15T23:58:49,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741890_1066 (size=12151) 2024-11-15T23:58:49,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715189800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:49,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715189810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:49,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:49,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:49,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 2024-11-15T23:58:50,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/62e19362358348a8a56ffe58bdf63c69 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69 2024-11-15T23:58:50,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69, entries=200, sequenceid=248, filesize=14.2 K 2024-11-15T23:58:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0cd11e9c2e6e42b086e5690172829c3f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f 2024-11-15T23:58:50,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T23:58:50,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 2024-11-15T23:58:50,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T23:58:50,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 131fcc9fbcad104b408318bd0c6e140c in 1465ms, sequenceid=248, compaction requested=false 2024-11-15T23:58:50,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:50,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T23:58:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:50,099 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:58:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c673af141f6a4cdd9fd353080c659d8e is 50, key is test_row_0/A:col10/1731715128624/Put/seqid=0 2024-11-15T23:58:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741891_1067 (size=12251) 2024-11-15T23:58:50,543 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c673af141f6a4cdd9fd353080c659d8e 2024-11-15T23:58:50,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1cc16e2d1575408b9b1bd05c9c442d8b is 50, key is test_row_0/B:col10/1731715128624/Put/seqid=0 2024-11-15T23:58:50,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741892_1068 (size=12251) 2024-11-15T23:58:50,575 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1cc16e2d1575408b9b1bd05c9c442d8b 2024-11-15T23:58:50,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/de4e25234343445f8e27b5366fc9ebc5 is 50, key is test_row_0/C:col10/1731715128624/Put/seqid=0 2024-11-15T23:58:50,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741893_1069 (size=12251) 2024-11-15T23:58:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715190928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715190935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715190979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715190980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:50,987 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:50,989 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:50,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715190991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:50,997 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 131fcc9fbcad104b408318bd0c6e140c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:58:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:50,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715191050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715191050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,073 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/de4e25234343445f8e27b5366fc9ebc5 2024-11-15T23:58:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c673af141f6a4cdd9fd353080c659d8e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e 2024-11-15T23:58:51,106 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T23:58:51,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1cc16e2d1575408b9b1bd05c9c442d8b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b 2024-11-15T23:58:51,117 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T23:58:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/de4e25234343445f8e27b5366fc9ebc5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5 2024-11-15T23:58:51,127 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T23:58:51,134 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 131fcc9fbcad104b408318bd0c6e140c in 1035ms, sequenceid=263, compaction requested=true 2024-11-15T23:58:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T23:58:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T23:58:51,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T23:58:51,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4590 sec 2024-11-15T23:58:51,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4710 sec 2024-11-15T23:58:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:51,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:51,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T23:58:51,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/886defdf899e4aaab235952b2a6acd1b is 50, key is test_row_0/A:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:51,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715191295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741894_1070 (size=14741) 2024-11-15T23:58:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715191302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715191410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715191420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715191621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715191630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/886defdf899e4aaab235952b2a6acd1b 2024-11-15T23:58:51,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0de3fd76f5fc4c1085bb265dc506da4a is 50, key is test_row_0/B:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:51,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741895_1071 (size=12301) 2024-11-15T23:58:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715191931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715191939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:52,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0de3fd76f5fc4c1085bb265dc506da4a 2024-11-15T23:58:52,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1dedd0a57fd4fc182ebd7eadf9269c2 is 50, key is test_row_0/C:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741896_1072 (size=12301) 2024-11-15T23:58:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715192440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:52,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715192450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:52,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:52,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:52,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:52,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1dedd0a57fd4fc182ebd7eadf9269c2 2024-11-15T23:58:52,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/886defdf899e4aaab235952b2a6acd1b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b 2024-11-15T23:58:52,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b, entries=200, sequenceid=286, filesize=14.4 K 2024-11-15T23:58:52,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0de3fd76f5fc4c1085bb265dc506da4a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a 2024-11-15T23:58:52,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T23:58:52,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/b1dedd0a57fd4fc182ebd7eadf9269c2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2 2024-11-15T23:58:52,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T23:58:52,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 131fcc9fbcad104b408318bd0c6e140c in 1372ms, sequenceid=286, compaction requested=true 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:52,637 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:52,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:52,637 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:52,639 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54230 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:52,639 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:52,639 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:52,640 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:52,640 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:52,640 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:52,640 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4f914197c4874aab89be6d0fb6de0a0a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=48.2 K 2024-11-15T23:58:52,640 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/25aadd1c0f644f059e4a4eef0aceddd8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=53.0 K 2024-11-15T23:58:52,641 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25aadd1c0f644f059e4a4eef0aceddd8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:52,641 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f914197c4874aab89be6d0fb6de0a0a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:52,642 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e19362358348a8a56ffe58bdf63c69, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715127361 2024-11-15T23:58:52,643 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd11e9c2e6e42b086e5690172829c3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715127361 2024-11-15T23:58:52,643 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c673af141f6a4cdd9fd353080c659d8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731715128624 2024-11-15T23:58:52,643 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc16e2d1575408b9b1bd05c9c442d8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731715128624 2024-11-15T23:58:52,644 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886defdf899e4aaab235952b2a6acd1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130922 2024-11-15T23:58:52,644 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de3fd76f5fc4c1085bb265dc506da4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130926 2024-11-15T23:58:52,664 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:52,664 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:52,665 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/9c1e3801d1dd4f42a72490c234f763e4 is 50, key is test_row_0/B:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:52,665 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/450e4e96bbcd47958d9283dbfda5f04e is 50, key is test_row_0/A:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741898_1074 (size=12983) 2024-11-15T23:58:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741897_1073 (size=12983) 2024-11-15T23:58:52,734 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/9c1e3801d1dd4f42a72490c234f763e4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/9c1e3801d1dd4f42a72490c234f763e4 2024-11-15T23:58:52,742 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into 9c1e3801d1dd4f42a72490c234f763e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:52,742 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:52,742 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=12, startTime=1731715132637; duration=0sec 2024-11-15T23:58:52,743 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:52,743 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:52,743 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:58:52,749 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:58:52,750 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:52,750 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:52,750 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e0041478d8ea4a49855087cfc4a5eada, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=48.2 K 2024-11-15T23:58:52,751 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e0041478d8ea4a49855087cfc4a5eada, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731715126770 2024-11-15T23:58:52,752 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fe6a0a4f0cd4fd78d86d3d9ca406bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715127361 2024-11-15T23:58:52,753 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting de4e25234343445f8e27b5366fc9ebc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731715128624 2024-11-15T23:58:52,754 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b1dedd0a57fd4fc182ebd7eadf9269c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130926 2024-11-15T23:58:52,779 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:52,780 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/007112c28b5841abb8f16b57e2856b47 is 50, key is test_row_0/C:col10/1731715130932/Put/seqid=0 2024-11-15T23:58:52,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741899_1075 (size=12983) 2024-11-15T23:58:52,819 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/007112c28b5841abb8f16b57e2856b47 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/007112c28b5841abb8f16b57e2856b47 2024-11-15T23:58:52,834 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 007112c28b5841abb8f16b57e2856b47(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:52,834 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:52,834 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=12, startTime=1731715132637; duration=0sec 2024-11-15T23:58:52,835 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:52,835 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T23:58:52,840 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:52,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T23:58:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:58:52,843 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:52,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:52,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:58:52,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T23:58:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,000 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:58:53,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ed7188eb66ad4385a21ee576af51a0de is 50, key is test_row_0/A:col10/1731715131293/Put/seqid=0 2024-11-15T23:58:53,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741900_1076 (size=12301) 2024-11-15T23:58:53,022 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ed7188eb66ad4385a21ee576af51a0de 2024-11-15T23:58:53,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/11d354f0612942939ebecdef021f7cca is 50, key is test_row_0/B:col10/1731715131293/Put/seqid=0 2024-11-15T23:58:53,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741901_1077 (size=12301) 2024-11-15T23:58:53,050 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/11d354f0612942939ebecdef021f7cca 2024-11-15T23:58:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/4da598b9df53475e82b5d9cad97d456f is 50, key is test_row_0/C:col10/1731715131293/Put/seqid=0 2024-11-15T23:58:53,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741902_1078 (size=12301) 2024-11-15T23:58:53,099 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/4da598b9df53475e82b5d9cad97d456f 2024-11-15T23:58:53,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ed7188eb66ad4385a21ee576af51a0de as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de 2024-11-15T23:58:53,127 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/450e4e96bbcd47958d9283dbfda5f04e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/450e4e96bbcd47958d9283dbfda5f04e 2024-11-15T23:58:53,127 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:58:53,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/11d354f0612942939ebecdef021f7cca as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca 2024-11-15T23:58:53,162 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:58:53,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/4da598b9df53475e82b5d9cad97d456f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f 2024-11-15T23:58:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:58:53,167 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 450e4e96bbcd47958d9283dbfda5f04e(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:53,167 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:53,167 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=12, startTime=1731715132637; duration=0sec 2024-11-15T23:58:53,167 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:53,167 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:53,179 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T23:58:53,181 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 131fcc9fbcad104b408318bd0c6e140c in 181ms, sequenceid=301, compaction requested=false 2024-11-15T23:58:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T23:58:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T23:58:53,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T23:58:53,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 340 msec 2024-11-15T23:58:53,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 349 msec 2024-11-15T23:58:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T23:58:53,495 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:53,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T23:58:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:53,500 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:53,501 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:53,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:53,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:58:53,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/2a2bfa79c57c4b81b7f5c9211083bed8 is 50, key is test_row_0/A:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:53,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741903_1079 (size=12301) 2024-11-15T23:58:53,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/2a2bfa79c57c4b81b7f5c9211083bed8 2024-11-15T23:58:53,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0381bb65a746484085c17d030aea93cb is 50, key is test_row_0/B:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:53,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741904_1080 (size=12301) 2024-11-15T23:58:53,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0381bb65a746484085c17d030aea93cb 2024-11-15T23:58:53,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715193596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:53,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715193598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:53,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/129099e913cc49d5ae685c232f4bbd32 is 50, key is test_row_0/C:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:53,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741905_1081 (size=12301) 2024-11-15T23:58:53,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/129099e913cc49d5ae685c232f4bbd32 2024-11-15T23:58:53,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T23:58:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:53,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/2a2bfa79c57c4b81b7f5c9211083bed8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8 2024-11-15T23:58:53,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T23:58:53,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/0381bb65a746484085c17d030aea93cb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb 2024-11-15T23:58:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715193712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715193713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:53,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T23:58:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/129099e913cc49d5ae685c232f4bbd32 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32 2024-11-15T23:58:53,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T23:58:53,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 131fcc9fbcad104b408318bd0c6e140c in 264ms, sequenceid=313, compaction requested=true 2024-11-15T23:58:53,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T23:58:53,774 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:53,777 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:53,798 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:53,798 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:53,798 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,798 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/450e4e96bbcd47958d9283dbfda5f04e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.7 K 2024-11-15T23:58:53,803 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:53,803 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:53,803 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450e4e96bbcd47958d9283dbfda5f04e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130926 2024-11-15T23:58:53,803 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,803 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/007112c28b5841abb8f16b57e2856b47, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.7 K 2024-11-15T23:58:53,809 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 007112c28b5841abb8f16b57e2856b47, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130926 2024-11-15T23:58:53,813 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed7188eb66ad4385a21ee576af51a0de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731715131282 2024-11-15T23:58:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:53,820 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da598b9df53475e82b5d9cad97d456f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731715131282 2024-11-15T23:58:53,820 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2bfa79c57c4b81b7f5c9211083bed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:53,829 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 129099e913cc49d5ae685c232f4bbd32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:53,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T23:58:53,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:53,831 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T23:58:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c490e8f8996940c2bc73e0fbf5119b51 is 50, key is test_row_0/A:col10/1731715133579/Put/seqid=0 2024-11-15T23:58:53,903 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:53,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741906_1082 (size=12301) 2024-11-15T23:58:53,904 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/fb8e4cfd6691420fb0dd00b5db6444a9 is 50, key is test_row_0/C:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:53,906 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:53,906 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/7c9d97e2adb04a4fad5005976a82bbe6 is 50, key is test_row_0/A:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:53,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741907_1083 (size=13085) 2024-11-15T23:58:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:53,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:53,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741908_1084 (size=13085) 2024-11-15T23:58:53,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715193969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:53,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715193965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715194090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715194090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:54,310 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c490e8f8996940c2bc73e0fbf5119b51 2024-11-15T23:58:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715194309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715194321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/059bc14f047740bfa8d27cbc7433d8eb is 50, key is test_row_0/B:col10/1731715133579/Put/seqid=0 2024-11-15T23:58:54,349 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/7c9d97e2adb04a4fad5005976a82bbe6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/7c9d97e2adb04a4fad5005976a82bbe6 2024-11-15T23:58:54,373 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/fb8e4cfd6691420fb0dd00b5db6444a9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fb8e4cfd6691420fb0dd00b5db6444a9 2024-11-15T23:58:54,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into 7c9d97e2adb04a4fad5005976a82bbe6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:54,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:54,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715133774; duration=0sec 2024-11-15T23:58:54,374 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:54,374 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:54,374 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:54,376 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:54,376 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:54,376 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:54,377 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/9c1e3801d1dd4f42a72490c234f763e4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.7 K 2024-11-15T23:58:54,377 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1e3801d1dd4f42a72490c234f763e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731715130926 2024-11-15T23:58:54,378 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d354f0612942939ebecdef021f7cca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731715131282 2024-11-15T23:58:54,379 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0381bb65a746484085c17d030aea93cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:54,381 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into fb8e4cfd6691420fb0dd00b5db6444a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:54,383 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:54,383 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715133774; duration=0sec 2024-11-15T23:58:54,383 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:54,383 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:54,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741909_1085 (size=12301) 2024-11-15T23:58:54,391 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:54,393 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/d33da98e262e49f08a1c7bf75ec3b9e0 is 50, key is test_row_0/B:col10/1731715133497/Put/seqid=0 2024-11-15T23:58:54,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741910_1086 (size=13085) 2024-11-15T23:58:54,455 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/d33da98e262e49f08a1c7bf75ec3b9e0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/d33da98e262e49f08a1c7bf75ec3b9e0 2024-11-15T23:58:54,465 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into d33da98e262e49f08a1c7bf75ec3b9e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:54,465 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:54,465 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715133774; duration=0sec 2024-11-15T23:58:54,466 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:54,466 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:54,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715194623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715194641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:54,785 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/059bc14f047740bfa8d27cbc7433d8eb 2024-11-15T23:58:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/fd116972b3df47afb8fec3fc30d86aff is 50, key is test_row_0/C:col10/1731715133579/Put/seqid=0 2024-11-15T23:58:54,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741911_1087 (size=12301) 2024-11-15T23:58:54,832 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/fd116972b3df47afb8fec3fc30d86aff 2024-11-15T23:58:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/c490e8f8996940c2bc73e0fbf5119b51 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51 2024-11-15T23:58:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51, entries=150, sequenceid=338, filesize=12.0 K 2024-11-15T23:58:54,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/059bc14f047740bfa8d27cbc7433d8eb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb 2024-11-15T23:58:54,871 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb, entries=150, sequenceid=338, filesize=12.0 K 2024-11-15T23:58:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/fd116972b3df47afb8fec3fc30d86aff as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff 2024-11-15T23:58:54,889 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff, entries=150, sequenceid=338, filesize=12.0 K 2024-11-15T23:58:54,891 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 131fcc9fbcad104b408318bd0c6e140c in 1060ms, sequenceid=338, compaction requested=false 2024-11-15T23:58:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T23:58:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T23:58:54,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T23:58:54,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-11-15T23:58:54,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4030 sec 2024-11-15T23:58:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:55,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:58:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:55,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/81f7c770c7d34aa88be02212cb3c4ed5 is 50, key is test_row_0/A:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:55,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741912_1088 (size=12301) 2024-11-15T23:58:55,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715195153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715195156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715195162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715195165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715195166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715195281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715195282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715195282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715195282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715195282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/81f7c770c7d34aa88be02212cb3c4ed5 2024-11-15T23:58:55,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/763c325e8979449aabad558b546d74ef is 50, key is test_row_0/B:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715195491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715195491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715195500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715195501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715195501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741913_1089 (size=12301) 2024-11-15T23:58:55,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/763c325e8979449aabad558b546d74ef 2024-11-15T23:58:55,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/26fe8914793640ac9efd38d05ea4e610 is 50, key is test_row_0/C:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:55,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741914_1090 (size=12301) 2024-11-15T23:58:55,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/26fe8914793640ac9efd38d05ea4e610 2024-11-15T23:58:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/81f7c770c7d34aa88be02212cb3c4ed5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5 2024-11-15T23:58:55,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T23:58:55,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/763c325e8979449aabad558b546d74ef as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef 2024-11-15T23:58:55,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T23:58:55,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/26fe8914793640ac9efd38d05ea4e610 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610 2024-11-15T23:58:55,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T23:58:55,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 131fcc9fbcad104b408318bd0c6e140c in 603ms, sequenceid=353, compaction requested=true 2024-11-15T23:58:55,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:55,614 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 131fcc9fbcad104b408318bd0c6e140c:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:58:55,614 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:55,616 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:55,616 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/B is initiating minor compaction (all files) 2024-11-15T23:58:55,616 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/B in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:55,617 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/d33da98e262e49f08a1c7bf75ec3b9e0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.8 K 2024-11-15T23:58:55,617 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:55,617 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/A is initiating minor compaction (all files) 2024-11-15T23:58:55,617 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/A in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:55,618 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/7c9d97e2adb04a4fad5005976a82bbe6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.8 K 2024-11-15T23:58:55,620 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d33da98e262e49f08a1c7bf75ec3b9e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:55,620 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9d97e2adb04a4fad5005976a82bbe6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:55,621 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 059bc14f047740bfa8d27cbc7433d8eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731715133579 2024-11-15T23:58:55,630 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c490e8f8996940c2bc73e0fbf5119b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731715133579 2024-11-15T23:58:55,630 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 763c325e8979449aabad558b546d74ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715133943 2024-11-15T23:58:55,630 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81f7c770c7d34aa88be02212cb3c4ed5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715133943 2024-11-15T23:58:55,647 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:55,648 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/b564449aafe446b29eab6bddc3db4061 is 50, key is test_row_0/A:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:55,654 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:55,654 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/94628ff7c59c4191a70b81eed4d5269c is 50, key is test_row_0/B:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T23:58:55,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:55,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:58:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T23:58:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:55,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741915_1091 (size=13187) 2024-11-15T23:58:55,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:58:55,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741916_1092 (size=13187) 2024-11-15T23:58:55,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:58:55,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:58:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:55,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:58:55,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T23:58:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:55,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ceaaad75b484478083fa1a3bb4e2b36e is 50, key is test_row_0/A:col10/1731715135163/Put/seqid=0 2024-11-15T23:58:55,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715195827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715195830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715195831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715195832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715195834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:55,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:55,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:55,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741917_1093 (size=12301) 2024-11-15T23:58:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715195941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715195941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715195950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715195950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715195951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:55,996 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T23:58:55,996 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:55,998 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T23:58:55,998 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,001 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T23:58:56,001 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,003 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,004 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,004 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,004 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,005 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,007 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,009 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T23:58:56,009 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,009 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,009 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,091 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/b564449aafe446b29eab6bddc3db4061 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b564449aafe446b29eab6bddc3db4061 2024-11-15T23:58:56,095 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/94628ff7c59c4191a70b81eed4d5269c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94628ff7c59c4191a70b81eed4d5269c 2024-11-15T23:58:56,101 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/A of 131fcc9fbcad104b408318bd0c6e140c into b564449aafe446b29eab6bddc3db4061(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:56,101 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:56,101 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/A, priority=13, startTime=1731715135614; duration=0sec 2024-11-15T23:58:56,101 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:58:56,101 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:A 2024-11-15T23:58:56,101 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:58:56,103 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:58:56,103 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 131fcc9fbcad104b408318bd0c6e140c/C is initiating minor compaction (all files) 2024-11-15T23:58:56,104 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 131fcc9fbcad104b408318bd0c6e140c/C in TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,104 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fb8e4cfd6691420fb0dd00b5db6444a9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp, totalSize=36.8 K 2024-11-15T23:58:56,104 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8e4cfd6691420fb0dd00b5db6444a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715133497 2024-11-15T23:58:56,105 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd116972b3df47afb8fec3fc30d86aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731715133579 2024-11-15T23:58:56,105 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26fe8914793640ac9efd38d05ea4e610, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715133943 2024-11-15T23:58:56,107 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/B of 131fcc9fbcad104b408318bd0c6e140c into 94628ff7c59c4191a70b81eed4d5269c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:56,107 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:56,107 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/B, priority=13, startTime=1731715135614; duration=0sec 2024-11-15T23:58:56,107 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:56,107 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:B 2024-11-15T23:58:56,119 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 131fcc9fbcad104b408318bd0c6e140c#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:58:56,119 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/99630ffafa544f4683e18248efec2fc3 is 50, key is test_row_0/C:col10/1731715133943/Put/seqid=0 2024-11-15T23:58:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741918_1094 (size=13187) 2024-11-15T23:58:56,149 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/99630ffafa544f4683e18248efec2fc3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/99630ffafa544f4683e18248efec2fc3 2024-11-15T23:58:56,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715196150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715196152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,158 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 131fcc9fbcad104b408318bd0c6e140c/C of 131fcc9fbcad104b408318bd0c6e140c into 99630ffafa544f4683e18248efec2fc3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:58:56,158 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:56,158 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., storeName=131fcc9fbcad104b408318bd0c6e140c/C, priority=13, startTime=1731715135614; duration=0sec 2024-11-15T23:58:56,159 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:58:56,159 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 131fcc9fbcad104b408318bd0c6e140c:C 2024-11-15T23:58:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715196161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715196171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715196171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:56,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ceaaad75b484478083fa1a3bb4e2b36e 2024-11-15T23:58:56,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1529590fb3ee4027b5112f709e3c0568 is 50, key is test_row_0/B:col10/1731715135163/Put/seqid=0 2024-11-15T23:58:56,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741919_1095 (size=12301) 2024-11-15T23:58:56,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1529590fb3ee4027b5112f709e3c0568 2024-11-15T23:58:56,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/eaf8aca0dd614223ab6b77fa94447c2e is 50, key is test_row_0/C:col10/1731715135163/Put/seqid=0 2024-11-15T23:58:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741920_1096 (size=12301) 2024-11-15T23:58:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51266 deadline: 1731715196460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51276 deadline: 1731715196460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1731715196470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51224 deadline: 1731715196480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:58:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51206 deadline: 1731715196491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 2024-11-15T23:58:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:58:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:58:56,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/eaf8aca0dd614223ab6b77fa94447c2e 2024-11-15T23:58:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/ceaaad75b484478083fa1a3bb4e2b36e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ceaaad75b484478083fa1a3bb4e2b36e 2024-11-15T23:58:56,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:58:56,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ceaaad75b484478083fa1a3bb4e2b36e, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T23:58:56,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/1529590fb3ee4027b5112f709e3c0568 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1529590fb3ee4027b5112f709e3c0568 2024-11-15T23:58:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1529590fb3ee4027b5112f709e3c0568, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T23:58:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/eaf8aca0dd614223ab6b77fa94447c2e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/eaf8aca0dd614223ab6b77fa94447c2e 2024-11-15T23:58:56,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/eaf8aca0dd614223ab6b77fa94447c2e, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T23:58:56,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 131fcc9fbcad104b408318bd0c6e140c in 1024ms, sequenceid=379, compaction requested=false 2024-11-15T23:58:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:56,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T23:58:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:56,967 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:58:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:58:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. as already flushing 2024-11-15T23:58:56,974 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T23:58:56,974 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,974 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,975 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,975 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T23:58:56,975 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,975 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,975 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,982 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T23:58:56,982 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,982 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,982 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f4df83f88d4c4093838ea6428c747662 is 50, key is test_row_0/A:col10/1731715135830/Put/seqid=0 2024-11-15T23:58:56,994 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T23:58:56,994 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:56,994 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:56,997 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:57,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:57,009 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T23:58:57,009 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:58:57,009 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:57,009 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:57,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741921_1097 (size=9857) 2024-11-15T23:58:57,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:57,410 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f4df83f88d4c4093838ea6428c747662 2024-11-15T23:58:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c1ac4fc235974e1782e601fcb685f801 is 50, key is test_row_0/B:col10/1731715135830/Put/seqid=0 2024-11-15T23:58:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741922_1098 (size=9857) 2024-11-15T23:58:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:57,830 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c1ac4fc235974e1782e601fcb685f801 2024-11-15T23:58:57,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 is 50, key is test_row_0/C:col10/1731715135830/Put/seqid=0 2024-11-15T23:58:57,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741923_1099 (size=9857) 2024-11-15T23:58:57,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:58:58,251 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 2024-11-15T23:58:58,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/f4df83f88d4c4093838ea6428c747662 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f4df83f88d4c4093838ea6428c747662 2024-11-15T23:58:58,264 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f4df83f88d4c4093838ea6428c747662, entries=100, sequenceid=393, filesize=9.6 K 2024-11-15T23:58:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c1ac4fc235974e1782e601fcb685f801 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c1ac4fc235974e1782e601fcb685f801 2024-11-15T23:58:58,271 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c1ac4fc235974e1782e601fcb685f801, entries=100, sequenceid=393, filesize=9.6 K 2024-11-15T23:58:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 2024-11-15T23:58:58,283 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/1d0cbb228afb4ab8a0a8eec93cf2cba0, entries=100, sequenceid=393, filesize=9.6 K 2024-11-15T23:58:58,284 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=33.54 KB/34350 for 131fcc9fbcad104b408318bd0c6e140c in 1317ms, sequenceid=393, compaction requested=true 2024-11-15T23:58:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 131fcc9fbcad104b408318bd0c6e140c: 2024-11-15T23:58:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:58:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T23:58:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T23:58:58,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T23:58:58,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5990 sec 2024-11-15T23:58:58,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.6280 sec 2024-11-15T23:58:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T23:58:59,840 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:58:59,840 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2359 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2387 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3397 rows 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-15T23:58:59,841 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3367 rows 2024-11-15T23:58:59,841 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:58:59,842 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:58:59,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:59,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:58:59,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:58:59,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T23:58:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:58:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:58:59,857 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715139856"}]},"ts":"1731715139856"} 2024-11-15T23:58:59,865 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:58:59,865 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:58:59,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:58:59,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, UNASSIGN}] 2024-11-15T23:58:59,872 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, UNASSIGN 2024-11-15T23:58:59,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=131fcc9fbcad104b408318bd0c6e140c, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:58:59,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, UNASSIGN because future has completed 2024-11-15T23:58:59,882 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:58:59,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:58:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:59:00,040 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:59:00,040 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:59:00,041 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 131fcc9fbcad104b408318bd0c6e140c, disabling compactions & flushes 2024-11-15T23:59:00,041 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:59:00,041 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:59:00,041 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. after waiting 0 ms 2024-11-15T23:59:00,041 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:59:00,041 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 131fcc9fbcad104b408318bd0c6e140c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T23:59:00,055 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fd462a68dc5d4d2fb3d7be415ce3df6d is 50, key is test_row_0/A:col10/1731715137006/Put/seqid=0 2024-11-15T23:59:00,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741924_1100 (size=7415) 2024-11-15T23:59:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:59:00,473 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fd462a68dc5d4d2fb3d7be415ce3df6d 2024-11-15T23:59:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:59:00,483 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c338cd3d994e4e4a8743c26e9e23f66f is 50, key is test_row_0/B:col10/1731715137006/Put/seqid=0 2024-11-15T23:59:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741925_1101 (size=7415) 2024-11-15T23:59:00,904 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c338cd3d994e4e4a8743c26e9e23f66f 2024-11-15T23:59:00,914 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/35ab1748a20b404a88d486ce4f45daba is 50, key is test_row_0/C:col10/1731715137006/Put/seqid=0 2024-11-15T23:59:00,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741926_1102 (size=7415) 2024-11-15T23:59:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:59:01,320 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/35ab1748a20b404a88d486ce4f45daba 2024-11-15T23:59:01,327 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/A/fd462a68dc5d4d2fb3d7be415ce3df6d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fd462a68dc5d4d2fb3d7be415ce3df6d 2024-11-15T23:59:01,333 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fd462a68dc5d4d2fb3d7be415ce3df6d, entries=50, sequenceid=401, filesize=7.2 K 2024-11-15T23:59:01,334 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/B/c338cd3d994e4e4a8743c26e9e23f66f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c338cd3d994e4e4a8743c26e9e23f66f 2024-11-15T23:59:01,339 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c338cd3d994e4e4a8743c26e9e23f66f, entries=50, sequenceid=401, filesize=7.2 K 2024-11-15T23:59:01,340 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/.tmp/C/35ab1748a20b404a88d486ce4f45daba as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/35ab1748a20b404a88d486ce4f45daba 2024-11-15T23:59:01,345 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/35ab1748a20b404a88d486ce4f45daba, entries=50, sequenceid=401, filesize=7.2 K 2024-11-15T23:59:01,346 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 131fcc9fbcad104b408318bd0c6e140c in 1305ms, sequenceid=401, compaction requested=true 2024-11-15T23:59:01,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9ca110e2c88f47f49c067e1e438e77bd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/278b503d16ec49aaba8c12338cc3fcd8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/36972fbeae014a26bf9e13684ed33bdf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d6cf6201424c4a9e908975423cd6e18f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/25aadd1c0f644f059e4a4eef0aceddd8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/450e4e96bbcd47958d9283dbfda5f04e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/7c9d97e2adb04a4fad5005976a82bbe6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5] to archive 2024-11-15T23:59:01,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:01,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b3773bd8fc0c418ea99bcfde9ae8a48e 2024-11-15T23:59:01,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fada121d5c5a4d568baabe4147aadf8d 2024-11-15T23:59:01,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9ca110e2c88f47f49c067e1e438e77bd to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9ca110e2c88f47f49c067e1e438e77bd 2024-11-15T23:59:01,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/9b5610bf0b954856ac739c97e3c72a9f 2024-11-15T23:59:01,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/3085dd39643242c6a604008c7d4fea16 2024-11-15T23:59:01,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/dc71f39482eb4fa6a2c03828594be56f 2024-11-15T23:59:01,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/278b503d16ec49aaba8c12338cc3fcd8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/278b503d16ec49aaba8c12338cc3fcd8 2024-11-15T23:59:01,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/44c0dc8912784e99b44fb37bd0d4af14 2024-11-15T23:59:01,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/5170b29cd8454bde836fb77889dccb59 2024-11-15T23:59:01,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/e4b2e2a67b964ac59b167f0d9f1e4553 2024-11-15T23:59:01,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/36972fbeae014a26bf9e13684ed33bdf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/36972fbeae014a26bf9e13684ed33bdf 2024-11-15T23:59:01,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/345bbdfca3c949e7b71612a0daaa03a0 2024-11-15T23:59:01,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f49b269a50f34204b74542f6edd63ccf 2024-11-15T23:59:01,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d6cf6201424c4a9e908975423cd6e18f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d6cf6201424c4a9e908975423cd6e18f 2024-11-15T23:59:01,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/356f0eba56394e43871be5d463fc625d 2024-11-15T23:59:01,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/d0d4dc08682a460399691d00c0893939 2024-11-15T23:59:01,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/25aadd1c0f644f059e4a4eef0aceddd8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/25aadd1c0f644f059e4a4eef0aceddd8 2024-11-15T23:59:01,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/62e19362358348a8a56ffe58bdf63c69 2024-11-15T23:59:01,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c673af141f6a4cdd9fd353080c659d8e 2024-11-15T23:59:01,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/886defdf899e4aaab235952b2a6acd1b 2024-11-15T23:59:01,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/450e4e96bbcd47958d9283dbfda5f04e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/450e4e96bbcd47958d9283dbfda5f04e 2024-11-15T23:59:01,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ed7188eb66ad4385a21ee576af51a0de 2024-11-15T23:59:01,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/7c9d97e2adb04a4fad5005976a82bbe6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/7c9d97e2adb04a4fad5005976a82bbe6 2024-11-15T23:59:01,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/2a2bfa79c57c4b81b7f5c9211083bed8 2024-11-15T23:59:01,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/c490e8f8996940c2bc73e0fbf5119b51 2024-11-15T23:59:01,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/81f7c770c7d34aa88be02212cb3c4ed5 2024-11-15T23:59:01,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e37e6bf1d639490083580e70e2bb902f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4cdf5afa7a3f47c28399f30531a28d17, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e73acb3825b342289051beac7336cbc3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/b4a973e3bf2e47c98525dde24fa6bad3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4f914197c4874aab89be6d0fb6de0a0a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/9c1e3801d1dd4f42a72490c234f763e4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/d33da98e262e49f08a1c7bf75ec3b9e0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef] to archive 2024-11-15T23:59:01,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:01,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dd19992bfc464778a2b0d8acb320a736 2024-11-15T23:59:01,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/da4a21fc02a041378e9237d749698898 2024-11-15T23:59:01,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e37e6bf1d639490083580e70e2bb902f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e37e6bf1d639490083580e70e2bb902f 2024-11-15T23:59:01,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94aa5c144e4f4313a880dba32191e4c9 2024-11-15T23:59:01,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/21e2be35e8994021a4793cd374c215ab 2024-11-15T23:59:01,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4cdf5afa7a3f47c28399f30531a28d17 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4cdf5afa7a3f47c28399f30531a28d17 2024-11-15T23:59:01,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3c4162519ef043db9fc590b5ab497806 2024-11-15T23:59:01,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e5ac1eabe0894d468bb79485a0553ebc 2024-11-15T23:59:01,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/dedf52705f3f4269b2eb0e8f370b51e5 2024-11-15T23:59:01,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e73acb3825b342289051beac7336cbc3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/e73acb3825b342289051beac7336cbc3 2024-11-15T23:59:01,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/a1b9ed2fe7e1411e81b2de22228d149e 2024-11-15T23:59:01,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/310185182ade4702a92802a5c130a1d9 2024-11-15T23:59:01,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/b4a973e3bf2e47c98525dde24fa6bad3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/b4a973e3bf2e47c98525dde24fa6bad3 2024-11-15T23:59:01,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/3fd1c804356c4782abe4db51912328aa 2024-11-15T23:59:01,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/5839d16b8fdb487ba5e5152d33e64163 2024-11-15T23:59:01,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4f914197c4874aab89be6d0fb6de0a0a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/4f914197c4874aab89be6d0fb6de0a0a 2024-11-15T23:59:01,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/defb180cd76948cb892bf997c3c8c937 2024-11-15T23:59:01,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0cd11e9c2e6e42b086e5690172829c3f 2024-11-15T23:59:01,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1cc16e2d1575408b9b1bd05c9c442d8b 2024-11-15T23:59:01,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/9c1e3801d1dd4f42a72490c234f763e4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/9c1e3801d1dd4f42a72490c234f763e4 2024-11-15T23:59:01,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0de3fd76f5fc4c1085bb265dc506da4a 2024-11-15T23:59:01,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/11d354f0612942939ebecdef021f7cca 2024-11-15T23:59:01,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/d33da98e262e49f08a1c7bf75ec3b9e0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/d33da98e262e49f08a1c7bf75ec3b9e0 2024-11-15T23:59:01,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/0381bb65a746484085c17d030aea93cb 2024-11-15T23:59:01,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/059bc14f047740bfa8d27cbc7433d8eb 2024-11-15T23:59:01,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/763c325e8979449aabad558b546d74ef 2024-11-15T23:59:01,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2efb554c503d482d8fd937fcd10f3ef2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/28262b4f3ff34b3e9df61aad3d70f7ca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/200c9f165773484392e10beee620d24d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/844fd459a0f84954aaa625a2a8cf8463, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e0041478d8ea4a49855087cfc4a5eada, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/007112c28b5841abb8f16b57e2856b47, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fb8e4cfd6691420fb0dd00b5db6444a9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610] to archive 2024-11-15T23:59:01,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:01,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/5410620719b44c9d9509c3d8405eedca 2024-11-15T23:59:01,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/d8f67fd57d154c19a3f5596ae2bbf069 2024-11-15T23:59:01,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2efb554c503d482d8fd937fcd10f3ef2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2efb554c503d482d8fd937fcd10f3ef2 2024-11-15T23:59:01,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/8772df749a514361b8193c490b38efdb 2024-11-15T23:59:01,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/f757a4f6c12a45bcba9f4308e1e172cb 2024-11-15T23:59:01,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/28262b4f3ff34b3e9df61aad3d70f7ca to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/28262b4f3ff34b3e9df61aad3d70f7ca 2024-11-15T23:59:01,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/7afb887114dd47e1ba0a794ed8d7bbbc 2024-11-15T23:59:01,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1452953f8fb47268882fb0348bbb6ab 2024-11-15T23:59:01,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/74aa8bea0c314831831718fb0b8db76c 2024-11-15T23:59:01,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/200c9f165773484392e10beee620d24d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/200c9f165773484392e10beee620d24d 2024-11-15T23:59:01,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e9763427841142ccb1cc7401b4b3e785 2024-11-15T23:59:01,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e04d5d5d00d749f0bebc0e4dda30b771 2024-11-15T23:59:01,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/844fd459a0f84954aaa625a2a8cf8463 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/844fd459a0f84954aaa625a2a8cf8463 2024-11-15T23:59:01,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/662fbe908ccf420491b2da0cdad22259 2024-11-15T23:59:01,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/15f15f7b677e43df83a06b905b7ed2a3 2024-11-15T23:59:01,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e0041478d8ea4a49855087cfc4a5eada to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/e0041478d8ea4a49855087cfc4a5eada 2024-11-15T23:59:01,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/ecbfbcf72d4d4e39bf7e011ece92a72e 2024-11-15T23:59:01,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/2fe6a0a4f0cd4fd78d86d3d9ca406bf6 2024-11-15T23:59:01,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/de4e25234343445f8e27b5366fc9ebc5 2024-11-15T23:59:01,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/007112c28b5841abb8f16b57e2856b47 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/007112c28b5841abb8f16b57e2856b47 2024-11-15T23:59:01,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/b1dedd0a57fd4fc182ebd7eadf9269c2 2024-11-15T23:59:01,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/4da598b9df53475e82b5d9cad97d456f 2024-11-15T23:59:01,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fb8e4cfd6691420fb0dd00b5db6444a9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fb8e4cfd6691420fb0dd00b5db6444a9 2024-11-15T23:59:01,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/129099e913cc49d5ae685c232f4bbd32 2024-11-15T23:59:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/fd116972b3df47afb8fec3fc30d86aff 2024-11-15T23:59:01,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/26fe8914793640ac9efd38d05ea4e610 2024-11-15T23:59:01,517 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/recovered.edits/404.seqid, newMaxSeqId=404, maxSeqId=1 2024-11-15T23:59:01,518 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. 2024-11-15T23:59:01,518 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 131fcc9fbcad104b408318bd0c6e140c: Waiting for close lock at 1731715140041Running coprocessor pre-close hooks at 1731715140041Disabling compacts and flushes for region at 1731715140041Disabling writes for close at 1731715140041Obtaining lock to block concurrent updates at 1731715140041Preparing flush snapshotting stores in 131fcc9fbcad104b408318bd0c6e140c at 1731715140041Finished memstore snapshotting TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731715140041Flushing stores of TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c. at 1731715140045 (+4 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/A: creating writer at 1731715140046 (+1 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/A: appending metadata at 1731715140054 (+8 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/A: closing flushed file at 1731715140055 (+1 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/B: creating writer at 1731715140479 (+424 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/B: appending metadata at 1731715140483 (+4 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/B: closing flushed file at 1731715140483Flushing 131fcc9fbcad104b408318bd0c6e140c/C: creating writer at 1731715140910 (+427 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/C: appending metadata at 1731715140913 (+3 ms)Flushing 131fcc9fbcad104b408318bd0c6e140c/C: closing flushed file at 1731715140913Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d3d8eed: reopening flushed file at 1731715141326 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d35711: reopening flushed file at 1731715141333 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e026849: reopening flushed file at 1731715141339 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 131fcc9fbcad104b408318bd0c6e140c in 1305ms, sequenceid=401, compaction requested=true at 1731715141346 (+7 ms)Writing region close event to WAL at 1731715141513 (+167 ms)Running coprocessor post-close hooks at 1731715141518 (+5 ms)Closed at 1731715141518 2024-11-15T23:59:01,521 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:59:01,523 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=131fcc9fbcad104b408318bd0c6e140c, regionState=CLOSED 2024-11-15T23:59:01,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:01,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T23:59:01,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 131fcc9fbcad104b408318bd0c6e140c, server=0801813e30ae,43013,1731715110098 in 1.6490 sec 2024-11-15T23:59:01,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T23:59:01,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=131fcc9fbcad104b408318bd0c6e140c, UNASSIGN in 1.6620 sec 2024-11-15T23:59:01,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T23:59:01,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6700 sec 2024-11-15T23:59:01,541 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715141541"}]},"ts":"1731715141541"} 2024-11-15T23:59:01,543 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:59:01,543 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:59:01,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6940 sec 2024-11-15T23:59:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T23:59:02,000 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:02,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T23:59:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,013 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T23:59:02,015 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,019 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:59:02,024 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/recovered.edits] 2024-11-15T23:59:02,028 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b564449aafe446b29eab6bddc3db4061 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/b564449aafe446b29eab6bddc3db4061 2024-11-15T23:59:02,032 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ceaaad75b484478083fa1a3bb4e2b36e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/ceaaad75b484478083fa1a3bb4e2b36e 2024-11-15T23:59:02,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f4df83f88d4c4093838ea6428c747662 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/f4df83f88d4c4093838ea6428c747662 2024-11-15T23:59:02,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fd462a68dc5d4d2fb3d7be415ce3df6d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/A/fd462a68dc5d4d2fb3d7be415ce3df6d 2024-11-15T23:59:02,039 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1529590fb3ee4027b5112f709e3c0568 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/1529590fb3ee4027b5112f709e3c0568 2024-11-15T23:59:02,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94628ff7c59c4191a70b81eed4d5269c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/94628ff7c59c4191a70b81eed4d5269c 2024-11-15T23:59:02,043 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c1ac4fc235974e1782e601fcb685f801 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c1ac4fc235974e1782e601fcb685f801 2024-11-15T23:59:02,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c338cd3d994e4e4a8743c26e9e23f66f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/B/c338cd3d994e4e4a8743c26e9e23f66f 2024-11-15T23:59:02,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/1d0cbb228afb4ab8a0a8eec93cf2cba0 2024-11-15T23:59:02,052 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/35ab1748a20b404a88d486ce4f45daba to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/35ab1748a20b404a88d486ce4f45daba 2024-11-15T23:59:02,054 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/99630ffafa544f4683e18248efec2fc3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/99630ffafa544f4683e18248efec2fc3 2024-11-15T23:59:02,057 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/eaf8aca0dd614223ab6b77fa94447c2e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/C/eaf8aca0dd614223ab6b77fa94447c2e 2024-11-15T23:59:02,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/recovered.edits/404.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c/recovered.edits/404.seqid 2024-11-15T23:59:02,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/131fcc9fbcad104b408318bd0c6e140c 2024-11-15T23:59:02,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:59:02,067 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T23:59:02,079 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:59:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T23:59:02,121 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:59:02,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:59:02,131 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715142130"}]},"ts":"9223372036854775807"} 2024-11-15T23:59:02,137 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:59:02,137 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 131fcc9fbcad104b408318bd0c6e140c, NAME => 'TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:59:02,137 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:59:02,138 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715142137"}]},"ts":"9223372036854775807"} 2024-11-15T23:59:02,142 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:59:02,146 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 140 msec 2024-11-15T23:59:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T23:59:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:59:02,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:02,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;0801813e30ae:43013-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/0801813e30ae:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1417717973_22 at /127.0.0.1:43986 [Waiting for operation #283] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1703801164_22 at /127.0.0.1:34578 [Waiting for operation #271] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=600 (was 479) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4864 (was 6431) 2024-11-15T23:59:02,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=600, ProcessCount=11, AvailableMemoryMB=4864 2024-11-15T23:59:02,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:59:02,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:59:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:02,359 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:59:02,360 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:02,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T23:59:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:59:02,361 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:59:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741927_1103 (size=959) 2024-11-15T23:59:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:59:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:59:02,776 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-15T23:59:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741928_1104 (size=53) 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 583360a8f892c9e569e274bdbe35bf48, disabling compactions & flushes 2024-11-15T23:59:02,788 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. after waiting 0 ms 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:02,788 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:02,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 583360a8f892c9e569e274bdbe35bf48: Waiting for close lock at 1731715142788Disabling compacts and flushes for region at 1731715142788Disabling writes for close at 1731715142788Writing region close event to WAL at 1731715142788Closed at 1731715142788 2024-11-15T23:59:02,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:59:02,790 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715142789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715142789"}]},"ts":"1731715142789"} 2024-11-15T23:59:02,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:59:02,796 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:59:02,796 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715142796"}]},"ts":"1731715142796"} 2024-11-15T23:59:02,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:59:02,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, ASSIGN}] 2024-11-15T23:59:02,800 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, ASSIGN 2024-11-15T23:59:02,801 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-15T23:59:02,952 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:02,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, ASSIGN because future has completed 2024-11-15T23:59:02,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:59:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:59:03,121 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:03,121 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:59:03,121 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,121 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:59:03,121 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,121 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,123 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,125 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName A 2024-11-15T23:59:03,125 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:03,126 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:03,126 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,134 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName B 2024-11-15T23:59:03,134 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:03,135 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:03,135 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,137 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName C 2024-11-15T23:59:03,137 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:03,138 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:03,138 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,139 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,139 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,141 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,141 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,143 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:59:03,144 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,147 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:59:03,149 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 583360a8f892c9e569e274bdbe35bf48; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67873912, jitterRate=0.011400103569030762}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:59:03,149 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:03,150 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 583360a8f892c9e569e274bdbe35bf48: Running coprocessor pre-open hook at 1731715143122Writing region info on filesystem at 1731715143122Initializing all the Stores at 1731715143122Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715143122Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715143123 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715143123Cleaning up temporary data from old regions at 1731715143141 (+18 ms)Running coprocessor post-open hooks at 1731715143149 (+8 ms)Region opened successfully at 1731715143150 (+1 ms) 2024-11-15T23:59:03,151 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., pid=30, masterSystemTime=1731715143108 2024-11-15T23:59:03,153 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:03,153 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:03,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:03,155 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38967 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=0801813e30ae,43013,1731715110098, table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T23:59:03,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:03,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T23:59:03,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 in 206 msec 2024-11-15T23:59:03,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T23:59:03,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, ASSIGN in 366 msec 2024-11-15T23:59:03,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:59:03,170 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715143169"}]},"ts":"1731715143169"} 2024-11-15T23:59:03,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:59:03,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:59:03,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 819 msec 2024-11-15T23:59:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T23:59:03,500 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:03,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdfec4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:03,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:03,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:03,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d97deaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:03,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a870147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:03,508 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:03,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:03,512 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:59:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-15T23:59:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5445cd2b 2024-11-15T23:59:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:59:03,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:59:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:59:03,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:59:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:03,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741929_1105 (size=995) 2024-11-15T23:59:03,940 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T23:59:03,940 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T23:59:03,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:59:03,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE}] 2024-11-15T23:59:03,954 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE 2024-11-15T23:59:03,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:03,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE because future has completed 2024-11-15T23:59:03,963 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:59:03,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:59:04,116 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,116 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:59:04,117 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 583360a8f892c9e569e274bdbe35bf48, disabling compactions & flushes 2024-11-15T23:59:04,117 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,117 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,117 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. after waiting 0 ms 2024-11-15T23:59:04,117 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,121 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T23:59:04,122 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,122 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 583360a8f892c9e569e274bdbe35bf48: Waiting for close lock at 1731715144116Running coprocessor pre-close hooks at 1731715144116Disabling compacts and flushes for region at 1731715144116Disabling writes for close at 1731715144117 (+1 ms)Writing region close event to WAL at 1731715144117Running coprocessor post-close hooks at 1731715144122 (+5 ms)Closed at 1731715144122 2024-11-15T23:59:04,122 WARN [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 583360a8f892c9e569e274bdbe35bf48 to self. 2024-11-15T23:59:04,124 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=CLOSED 2024-11-15T23:59:04,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:04,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T23:59:04,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 in 165 msec 2024-11-15T23:59:04,131 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE; state=CLOSED, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=true 2024-11-15T23:59:04,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE because future has completed 2024-11-15T23:59:04,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:59:04,439 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,439 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:59:04,439 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,439 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:59:04,440 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,440 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,442 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,447 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName A 2024-11-15T23:59:04,449 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:04,450 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:04,450 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,451 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName B 2024-11-15T23:59:04,451 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:04,452 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:04,452 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,452 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 583360a8f892c9e569e274bdbe35bf48 columnFamilyName C 2024-11-15T23:59:04,453 DEBUG [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:04,453 INFO [StoreOpener-583360a8f892c9e569e274bdbe35bf48-1 {}] regionserver.HStore(327): Store=583360a8f892c9e569e274bdbe35bf48/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:04,453 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,454 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,455 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,455 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,455 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,456 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:59:04,457 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,458 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 583360a8f892c9e569e274bdbe35bf48; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64405166, jitterRate=-0.04028823971748352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:59:04,458 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,460 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 583360a8f892c9e569e274bdbe35bf48: Running coprocessor pre-open hook at 1731715144440Writing region info on filesystem at 1731715144440Initializing all the Stores at 1731715144440Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715144440Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715144441 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715144441Cleaning up temporary data from old regions at 1731715144456 (+15 ms)Running coprocessor post-open hooks at 1731715144458 (+2 ms)Region opened successfully at 1731715144460 (+2 ms) 2024-11-15T23:59:04,461 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., pid=35, masterSystemTime=1731715144436 2024-11-15T23:59:04,463 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,463 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,464 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=OPEN, openSeqNum=5, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:04,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T23:59:04,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 in 183 msec 2024-11-15T23:59:04,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T23:59:04,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, REOPEN/MOVE in 516 msec 2024-11-15T23:59:04,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T23:59:04,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-15T23:59:04,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-15T23:59:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T23:59:04,480 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:04,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dba4ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,482 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431422c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,485 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c00921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,487 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678171c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcab5b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b1860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c370f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b861e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22aa04a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c612c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c93246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52903b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,502 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5597fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b149f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d914c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,509 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3670f44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48205543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,514 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3174ac96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,519 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b794f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,522 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4fb41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf07c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:04,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:04,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cb153b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:04,532 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:04,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296fe578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:04,538 DEBUG [Thread-565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,538 DEBUG [Thread-558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,539 DEBUG [Thread-561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T23:59:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,541 DEBUG [Thread-563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:04,542 DEBUG [Thread-567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,544 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:04,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,545 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:04,546 DEBUG [Thread-554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:04,546 DEBUG [Thread-550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,548 DEBUG [Thread-556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,548 DEBUG [Thread-552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:04,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-15T23:59:04,579 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:04,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:59:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d4943868f65471ca9e40426ec0f6f76_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715144560/Put/seqid=0 2024-11-15T23:59:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715204648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715204652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715204653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715204659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741930_1106 (size=12154) 2024-11-15T23:59:04,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715204656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:04,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:04,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:04,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d4943868f65471ca9e40426ec0f6f76_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d4943868f65471ca9e40426ec0f6f76_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:04,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/cef29feaa32248ca850d3d64b5667649, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:04,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/cef29feaa32248ca850d3d64b5667649 is 175, key is test_row_0/A:col10/1731715144560/Put/seqid=0 2024-11-15T23:59:04,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741931_1107 (size=30955) 2024-11-15T23:59:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715204770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715204781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715204781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715204781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715204781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:04,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:04,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715204993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715204994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715204994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715205000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715204994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/cef29feaa32248ca850d3d64b5667649 2024-11-15T23:59:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:05,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/da3461a0d1bc4005b9ab03289e90ecdf is 50, key is test_row_0/B:col10/1731715144560/Put/seqid=0 2024-11-15T23:59:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741932_1108 (size=12001) 2024-11-15T23:59:05,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/da3461a0d1bc4005b9ab03289e90ecdf 2024-11-15T23:59:05,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 is 50, key is test_row_0/C:col10/1731715144560/Put/seqid=0 2024-11-15T23:59:05,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741933_1109 (size=12001) 2024-11-15T23:59:05,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715205310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715205310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715205310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715205311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715205311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:05,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 2024-11-15T23:59:05,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/cef29feaa32248ca850d3d64b5667649 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649 2024-11-15T23:59:05,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T23:59:05,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/da3461a0d1bc4005b9ab03289e90ecdf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf 2024-11-15T23:59:05,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T23:59:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 2024-11-15T23:59:05,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T23:59:05,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 583360a8f892c9e569e274bdbe35bf48 in 1192ms, sequenceid=16, compaction requested=false 2024-11-15T23:59:05,771 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T23:59:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:05,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T23:59:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:05,833 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154541372ba2ae4879b42064ca4fccfc7e_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715144652/Put/seqid=0 2024-11-15T23:59:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:05,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:05,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715205871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715205878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715205882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741934_1110 (size=12154) 2024-11-15T23:59:05,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715205890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,913 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154541372ba2ae4879b42064ca4fccfc7e_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154541372ba2ae4879b42064ca4fccfc7e_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:05,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715205889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:05,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5febf44d30184f8893554be56d846b73, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:05,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5febf44d30184f8893554be56d846b73 is 175, key is test_row_0/A:col10/1731715144652/Put/seqid=0 2024-11-15T23:59:05,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741935_1111 (size=30955) 2024-11-15T23:59:05,961 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5febf44d30184f8893554be56d846b73 2024-11-15T23:59:05,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/03cb711e999b459cadcdea3792af9304 is 50, key is test_row_0/B:col10/1731715144652/Put/seqid=0 2024-11-15T23:59:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715206000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715206000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715206000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741936_1112 (size=12001) 2024-11-15T23:59:06,015 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/03cb711e999b459cadcdea3792af9304 2024-11-15T23:59:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715206011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715206020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c78c67fc9d294c5d9b37b2c1b49c974a is 50, key is test_row_0/C:col10/1731715144652/Put/seqid=0 2024-11-15T23:59:06,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741937_1113 (size=12001) 2024-11-15T23:59:06,063 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c78c67fc9d294c5d9b37b2c1b49c974a 2024-11-15T23:59:06,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5febf44d30184f8893554be56d846b73 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73 2024-11-15T23:59:06,083 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T23:59:06,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/03cb711e999b459cadcdea3792af9304 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304 2024-11-15T23:59:06,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T23:59:06,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c78c67fc9d294c5d9b37b2c1b49c974a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a 2024-11-15T23:59:06,118 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T23:59:06,119 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 583360a8f892c9e569e274bdbe35bf48 in 286ms, sequenceid=40, compaction requested=false 2024-11-15T23:59:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T23:59:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T23:59:06,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T23:59:06,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-15T23:59:06,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.5860 sec 2024-11-15T23:59:06,220 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:06,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:59:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:06,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fe04cfcde57a46e491592e5107e87778_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715206263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715206264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741938_1114 (size=12154) 2024-11-15T23:59:06,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715206266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715206268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715206269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fe04cfcde57a46e491592e5107e87778_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe04cfcde57a46e491592e5107e87778_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:06,303 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/a97a360811b64c1e871e921cc4cdc68b, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/a97a360811b64c1e871e921cc4cdc68b is 175, key is test_row_0/A:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741939_1115 (size=30955) 2024-11-15T23:59:06,338 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/a97a360811b64c1e871e921cc4cdc68b 2024-11-15T23:59:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715206382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ea2cef52186745568f90865a214d8599 is 50, key is test_row_0/B:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715206391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715206391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715206392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715206393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741940_1116 (size=12001) 2024-11-15T23:59:06,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ea2cef52186745568f90865a214d8599 2024-11-15T23:59:06,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e619a576d1e2451b8acd7795931e2f78 is 50, key is test_row_0/C:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741941_1117 (size=12001) 2024-11-15T23:59:06,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e619a576d1e2451b8acd7795931e2f78 2024-11-15T23:59:06,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/a97a360811b64c1e871e921cc4cdc68b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b 2024-11-15T23:59:06,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b, entries=150, sequenceid=54, filesize=30.2 K 2024-11-15T23:59:06,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ea2cef52186745568f90865a214d8599 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599 2024-11-15T23:59:06,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599, entries=150, sequenceid=54, filesize=11.7 K 2024-11-15T23:59:06,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e619a576d1e2451b8acd7795931e2f78 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78 2024-11-15T23:59:06,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715206592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78, entries=150, sequenceid=54, filesize=11.7 K 2024-11-15T23:59:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715206601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 583360a8f892c9e569e274bdbe35bf48 in 390ms, sequenceid=54, compaction requested=true 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:06,610 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:06,610 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:06,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:06,612 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:06,612 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:06,612 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,612 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=90.7 K 2024-11-15T23:59:06,612 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,613 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b] 2024-11-15T23:59:06,613 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:06,613 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:06,613 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,613 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.2 K 2024-11-15T23:59:06,617 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef29feaa32248ca850d3d64b5667649, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715144560 2024-11-15T23:59:06,617 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting da3461a0d1bc4005b9ab03289e90ecdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715144560 2024-11-15T23:59:06,617 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5febf44d30184f8893554be56d846b73, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715144650 2024-11-15T23:59:06,618 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 03cb711e999b459cadcdea3792af9304, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715144650 2024-11-15T23:59:06,618 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97a360811b64c1e871e921cc4cdc68b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:06,619 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2cef52186745568f90865a214d8599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:06,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:06,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:06,635 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,643 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#100 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:06,643 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/6bb01306fcda40119e5d67cba2ef706b is 50, key is test_row_0/B:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,650 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115d576add2b5c24595bcff89fe58941393_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,654 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115d576add2b5c24595bcff89fe58941393_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,655 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d576add2b5c24595bcff89fe58941393_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715206652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715206651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715206665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c4413324c7942589ed02fc194382ce2_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715146614/Put/seqid=0 2024-11-15T23:59:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T23:59:06,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:06,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T23:59:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:06,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:06,713 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:06,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:06,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741942_1118 (size=12104) 2024-11-15T23:59:06,735 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/6bb01306fcda40119e5d67cba2ef706b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/6bb01306fcda40119e5d67cba2ef706b 2024-11-15T23:59:06,743 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 6bb01306fcda40119e5d67cba2ef706b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:06,743 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,743 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715146610; duration=0sec 2024-11-15T23:59:06,744 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:06,744 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:06,744 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:06,745 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:06,745 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:06,745 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,746 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.2 K 2024-11-15T23:59:06,746 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d755ee1eb7cd42b58d7eb0062f9bf0e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715144560 2024-11-15T23:59:06,747 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c78c67fc9d294c5d9b37b2c1b49c974a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715144650 2024-11-15T23:59:06,747 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e619a576d1e2451b8acd7795931e2f78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:06,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741943_1119 (size=4469) 2024-11-15T23:59:06,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741944_1120 (size=12154) 2024-11-15T23:59:06,755 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#99 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:06,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:06,757 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/d4faf6ce1b14469798acede067f63591 is 175, key is test_row_0/A:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c4413324c7942589ed02fc194382ce2_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4413324c7942589ed02fc194382ce2_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:06,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715206772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715206773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/90d42a69deaa44158e0e539770d59495, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/90d42a69deaa44158e0e539770d59495 is 175, key is test_row_0/A:col10/1731715146614/Put/seqid=0 2024-11-15T23:59:06,782 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:06,783 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8f5cae6fb0d24515bdd3840003dfd5e4 is 50, key is test_row_0/C:col10/1731715146215/Put/seqid=0 2024-11-15T23:59:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715206780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:06,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741945_1121 (size=31058) 2024-11-15T23:59:06,831 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/d4faf6ce1b14469798acede067f63591 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591 2024-11-15T23:59:06,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741946_1122 (size=30955) 2024-11-15T23:59:06,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/90d42a69deaa44158e0e539770d59495 2024-11-15T23:59:06,844 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into d4faf6ce1b14469798acede067f63591(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:06,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,844 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715146610; duration=0sec 2024-11-15T23:59:06,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:06,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:06,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/67481dee91b9439ba8d7cc35e79ce3fb is 50, key is test_row_0/B:col10/1731715146614/Put/seqid=0 2024-11-15T23:59:06,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741947_1123 (size=12104) 2024-11-15T23:59:06,871 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8f5cae6fb0d24515bdd3840003dfd5e4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8f5cae6fb0d24515bdd3840003dfd5e4 2024-11-15T23:59:06,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:06,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:06,881 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into 8f5cae6fb0d24515bdd3840003dfd5e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:06,881 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,881 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715146610; duration=0sec 2024-11-15T23:59:06,881 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:06,881 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:06,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741948_1124 (size=12001) 2024-11-15T23:59:06,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/67481dee91b9439ba8d7cc35e79ce3fb 2024-11-15T23:59:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/492dd168911b42618a96a701af5d1381 is 50, key is test_row_0/C:col10/1731715146614/Put/seqid=0 2024-11-15T23:59:06,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715206910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715206920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:06,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:06,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741949_1125 (size=12001) 2024-11-15T23:59:06,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/492dd168911b42618a96a701af5d1381 2024-11-15T23:59:06,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/90d42a69deaa44158e0e539770d59495 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495 2024-11-15T23:59:06,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495, entries=150, sequenceid=80, filesize=30.2 K 2024-11-15T23:59:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/67481dee91b9439ba8d7cc35e79ce3fb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb 2024-11-15T23:59:06,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T23:59:06,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/492dd168911b42618a96a701af5d1381 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381 2024-11-15T23:59:06,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T23:59:06,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 583360a8f892c9e569e274bdbe35bf48 in 365ms, sequenceid=80, compaction requested=false 2024-11-15T23:59:06,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:06,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:07,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bc44fdb04e9b4c26a924a50857d97296_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:07,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741950_1126 (size=12154) 2024-11-15T23:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,079 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:59:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715207095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715207097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715207097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715207212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715207220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715207220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:07,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715207430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715207430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715207433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,443 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:07,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715207441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715207442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bc44fdb04e9b4c26a924a50857d97296_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc44fdb04e9b4c26a924a50857d97296_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:07,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/0a453cb244164dc4902d1eae6a8c8794, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/0a453cb244164dc4902d1eae6a8c8794 is 175, key is test_row_0/A:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741951_1127 (size=30955) 2024-11-15T23:59:07,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/0a453cb244164dc4902d1eae6a8c8794 2024-11-15T23:59:07,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/68e0182558b14b0ba6dcfb1ea26c647c is 50, key is test_row_0/B:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741952_1128 (size=12001) 2024-11-15T23:59:07,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/68e0182558b14b0ba6dcfb1ea26c647c 2024-11-15T23:59:07,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/03185af5aa174e80ba69c0aff6200a74 is 50, key is test_row_0/C:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741953_1129 (size=12001) 2024-11-15T23:59:07,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/03185af5aa174e80ba69c0aff6200a74 2024-11-15T23:59:07,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/0a453cb244164dc4902d1eae6a8c8794 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794 2024-11-15T23:59:07,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T23:59:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/68e0182558b14b0ba6dcfb1ea26c647c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c 2024-11-15T23:59:07,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T23:59:07,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/03185af5aa174e80ba69c0aff6200a74 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74 2024-11-15T23:59:07,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T23:59:07,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 583360a8f892c9e569e274bdbe35bf48 in 732ms, sequenceid=95, compaction requested=true 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:07,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:59:07,727 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:07,731 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:07,731 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:07,732 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,732 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=90.8 K 2024-11-15T23:59:07,732 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,732 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794] 2024-11-15T23:59:07,732 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:07,732 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:07,732 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,732 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/6bb01306fcda40119e5d67cba2ef706b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.3 K 2024-11-15T23:59:07,733 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d4faf6ce1b14469798acede067f63591, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:07,733 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb01306fcda40119e5d67cba2ef706b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:07,733 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d42a69deaa44158e0e539770d59495, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731715146614 2024-11-15T23:59:07,734 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67481dee91b9439ba8d7cc35e79ce3fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731715146614 2024-11-15T23:59:07,734 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a453cb244164dc4902d1eae6a8c8794, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:07,734 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e0182558b14b0ba6dcfb1ea26c647c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:07,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:07,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T23:59:07,759 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:07,760 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/11d4a633f3d748eea7403c9451686f11 is 50, key is test_row_0/B:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,770 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715207769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715207772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715207776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,798 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154576a8502b5644baa82041ba1c71c832_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,803 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154576a8502b5644baa82041ba1c71c832_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,803 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154576a8502b5644baa82041ba1c71c832_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d307fa397d67414eaa88b7b5e7d3d6b6_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715147094/Put/seqid=0 2024-11-15T23:59:07,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741954_1130 (size=12207) 2024-11-15T23:59:07,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,849 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/11d4a633f3d748eea7403c9451686f11 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/11d4a633f3d748eea7403c9451686f11 2024-11-15T23:59:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:07,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741955_1131 (size=4469) 2024-11-15T23:59:07,857 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#109 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:07,858 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/25689089e92b4da39dd9fd20ddd5b27d is 175, key is test_row_0/A:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,858 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 11d4a633f3d748eea7403c9451686f11(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:07,858 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:07,858 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715147727; duration=0sec 2024-11-15T23:59:07,858 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:07,858 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:07,858 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:07,863 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:07,863 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:07,863 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,863 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8f5cae6fb0d24515bdd3840003dfd5e4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.3 K 2024-11-15T23:59:07,864 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5cae6fb0d24515bdd3840003dfd5e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715145879 2024-11-15T23:59:07,864 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492dd168911b42618a96a701af5d1381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731715146614 2024-11-15T23:59:07,865 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03185af5aa174e80ba69c0aff6200a74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:07,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741956_1132 (size=14594) 2024-11-15T23:59:07,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:07,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d307fa397d67414eaa88b7b5e7d3d6b6_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d307fa397d67414eaa88b7b5e7d3d6b6_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:07,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/6a7c7fd306d24141ad832233f0890125, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:07,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/6a7c7fd306d24141ad832233f0890125 is 175, key is test_row_0/A:col10/1731715147094/Put/seqid=0 2024-11-15T23:59:07,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715207892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715207892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715207901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:07,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:07,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741957_1133 (size=31161) 2024-11-15T23:59:07,921 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:07,922 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/ea92efceaf84487b95ccd704827c2c54 is 50, key is test_row_0/C:col10/1731715146637/Put/seqid=0 2024-11-15T23:59:07,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741958_1134 (size=39549) 2024-11-15T23:59:07,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/6a7c7fd306d24141ad832233f0890125 2024-11-15T23:59:07,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741959_1135 (size=12207) 2024-11-15T23:59:07,985 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/ea92efceaf84487b95ccd704827c2c54 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/ea92efceaf84487b95ccd704827c2c54 2024-11-15T23:59:07,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/1739e759bb3e41c8b7d0563a6100e492 is 50, key is test_row_0/B:col10/1731715147094/Put/seqid=0 2024-11-15T23:59:07,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:07,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,995 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into ea92efceaf84487b95ccd704827c2c54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:07,995 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:07,995 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715147727; duration=0sec 2024-11-15T23:59:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:07,995 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:07,995 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741960_1136 (size=12001) 2024-11-15T23:59:08,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/1739e759bb3e41c8b7d0563a6100e492 2024-11-15T23:59:08,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/08d24673a9e7414b9328c0b127e77893 is 50, key is test_row_0/C:col10/1731715147094/Put/seqid=0 2024-11-15T23:59:08,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741961_1137 (size=12001) 2024-11-15T23:59:08,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/08d24673a9e7414b9328c0b127e77893 2024-11-15T23:59:08,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/6a7c7fd306d24141ad832233f0890125 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125 2024-11-15T23:59:08,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125, entries=200, sequenceid=120, filesize=38.6 K 2024-11-15T23:59:08,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/1739e759bb3e41c8b7d0563a6100e492 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492 2024-11-15T23:59:08,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715208100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715208105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T23:59:08,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/08d24673a9e7414b9328c0b127e77893 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893 2024-11-15T23:59:08,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715208110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T23:59:08,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 583360a8f892c9e569e274bdbe35bf48 in 376ms, sequenceid=120, compaction requested=false 2024-11-15T23:59:08,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:08,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T23:59:08,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:08,149 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:08,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4f90509c07a44e49dd02530bad11b5f_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715147768/Put/seqid=0 2024-11-15T23:59:08,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741962_1138 (size=12254) 2024-11-15T23:59:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:08,214 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4f90509c07a44e49dd02530bad11b5f_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4f90509c07a44e49dd02530bad11b5f_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:08,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c90c56d11ca44a78296fe8e6e932d18, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c90c56d11ca44a78296fe8e6e932d18 is 175, key is test_row_0/A:col10/1731715147768/Put/seqid=0 2024-11-15T23:59:08,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741963_1139 (size=31055) 2024-11-15T23:59:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c90c56d11ca44a78296fe8e6e932d18 2024-11-15T23:59:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/feb3a2a254bd431dba7c54f80cbe8b27 is 50, key is test_row_0/B:col10/1731715147768/Put/seqid=0 2024-11-15T23:59:08,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741964_1140 (size=12101) 2024-11-15T23:59:08,350 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/25689089e92b4da39dd9fd20ddd5b27d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d 2024-11-15T23:59:08,354 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/feb3a2a254bd431dba7c54f80cbe8b27 2024-11-15T23:59:08,406 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into 25689089e92b4da39dd9fd20ddd5b27d(size=30.4 K), total size for store is 69.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:08,406 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:08,406 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715147727; duration=0sec 2024-11-15T23:59:08,406 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:08,406 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:08,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/edd907637f5b4721b4454b8e088a233f is 50, key is test_row_0/C:col10/1731715147768/Put/seqid=0 2024-11-15T23:59:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:08,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:08,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741965_1141 (size=12101) 2024-11-15T23:59:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715208541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715208543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715208544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715208547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715208548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715208660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715208660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715208660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715208661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715208661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,867 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/edd907637f5b4721b4454b8e088a233f 2024-11-15T23:59:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:08,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715208870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c90c56d11ca44a78296fe8e6e932d18 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18 2024-11-15T23:59:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715208871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715208871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,880 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18, entries=150, sequenceid=133, filesize=30.3 K 2024-11-15T23:59:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/feb3a2a254bd431dba7c54f80cbe8b27 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27 2024-11-15T23:59:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715208880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715208880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:08,889 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27, entries=150, sequenceid=133, filesize=11.8 K 2024-11-15T23:59:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/edd907637f5b4721b4454b8e088a233f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f 2024-11-15T23:59:08,900 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f, entries=150, sequenceid=133, filesize=11.8 K 2024-11-15T23:59:08,905 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 583360a8f892c9e569e274bdbe35bf48 in 756ms, sequenceid=133, compaction requested=true 2024-11-15T23:59:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:08,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T23:59:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T23:59:08,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T23:59:08,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1970 sec 2024-11-15T23:59:08,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.2130 sec 2024-11-15T23:59:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T23:59:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:09,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715209193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715209195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715209195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715209197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715209201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a36fe619c8134ec0aa5fb97706cc97ee_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:09,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741966_1142 (size=17284) 2024-11-15T23:59:09,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a36fe619c8134ec0aa5fb97706cc97ee_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a36fe619c8134ec0aa5fb97706cc97ee_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:09,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5d213e13f4c141e290ba512132cfda70, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:09,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5d213e13f4c141e290ba512132cfda70 is 175, key is test_row_0/A:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:09,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741967_1143 (size=48389) 2024-11-15T23:59:09,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715209299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715209313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715209313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715209313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715209512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715209534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715209535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715209536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5d213e13f4c141e290ba512132cfda70 2024-11-15T23:59:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715209735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/32f6f180f96044b6a9363b44ff131984 is 50, key is test_row_0/B:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:09,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741968_1144 (size=12151) 2024-11-15T23:59:09,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/32f6f180f96044b6a9363b44ff131984 2024-11-15T23:59:09,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/02d70f0b6bef4c0ab88f4e0106d4e196 is 50, key is test_row_0/C:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:09,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715209820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715209841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715209850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715209850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741969_1145 (size=12151) 2024-11-15T23:59:09,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/02d70f0b6bef4c0ab88f4e0106d4e196 2024-11-15T23:59:09,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5d213e13f4c141e290ba512132cfda70 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70 2024-11-15T23:59:09,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70, entries=250, sequenceid=162, filesize=47.3 K 2024-11-15T23:59:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/32f6f180f96044b6a9363b44ff131984 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984 2024-11-15T23:59:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984, entries=150, sequenceid=162, filesize=11.9 K 2024-11-15T23:59:09,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/02d70f0b6bef4c0ab88f4e0106d4e196 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196 2024-11-15T23:59:09,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196, entries=150, sequenceid=162, filesize=11.9 K 2024-11-15T23:59:09,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 583360a8f892c9e569e274bdbe35bf48 in 755ms, sequenceid=162, compaction requested=true 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:09,940 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:09,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:59:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,943 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,944 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:09,944 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:09,944 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:09,944 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=146.6 K 2024-11-15T23:59:09,944 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:09,944 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70] 2024-11-15T23:59:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,948 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25689089e92b4da39dd9fd20ddd5b27d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:09,948 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:09,948 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:09,948 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,948 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/11d4a633f3d748eea7403c9451686f11, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=47.3 K 2024-11-15T23:59:09,949 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7c7fd306d24141ad832233f0890125, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731715147048 2024-11-15T23:59:09,949 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d4a633f3d748eea7403c9451686f11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:09,950 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c90c56d11ca44a78296fe8e6e932d18, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731715147754 2024-11-15T23:59:09,950 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1739e759bb3e41c8b7d0563a6100e492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731715147092 2024-11-15T23:59:09,951 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d213e13f4c141e290ba512132cfda70, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148470 2024-11-15T23:59:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,951 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting feb3a2a254bd431dba7c54f80cbe8b27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731715147754 2024-11-15T23:59:09,952 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 32f6f180f96044b6a9363b44ff131984, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148543 2024-11-15T23:59:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,975 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,989 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111558ec62185c3949dda86389983a48fa96_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,007 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111558ec62185c3949dda86389983a48fa96_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,007 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111558ec62185c3949dda86389983a48fa96_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:10,008 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#121 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,009 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/b2ba08546de24909b88baafff87f72c7 is 50, key is test_row_0/B:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741970_1146 (size=4469) 2024-11-15T23:59:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,050 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#120 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,051 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/9bbcf32cc0224295b0782e0907439e1d is 175, key is test_row_0/A:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741971_1147 (size=12493) 2024-11-15T23:59:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741972_1148 (size=31447) 2024-11-15T23:59:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:10,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:10,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d659a1178ef84a9e8e7cafe439368f57_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715150382/Put/seqid=0 2024-11-15T23:59:10,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741973_1149 (size=19774) 2024-11-15T23:59:10,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:10,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715210481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715210483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,492 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d659a1178ef84a9e8e7cafe439368f57_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d659a1178ef84a9e8e7cafe439368f57_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:10,493 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3d7e826ecb654ee8aa4b544e7f08a04a, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:10,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3d7e826ecb654ee8aa4b544e7f08a04a is 175, key is test_row_0/A:col10/1731715150382/Put/seqid=0 2024-11-15T23:59:10,494 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/b2ba08546de24909b88baafff87f72c7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/b2ba08546de24909b88baafff87f72c7 2024-11-15T23:59:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715210488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715210488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,501 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into b2ba08546de24909b88baafff87f72c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:10,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:10,501 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=12, startTime=1731715149940; duration=0sec 2024-11-15T23:59:10,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:10,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:10,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:10,503 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:10,503 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:10,503 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:10,504 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/ea92efceaf84487b95ccd704827c2c54, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=47.3 K 2024-11-15T23:59:10,504 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ea92efceaf84487b95ccd704827c2c54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731715146637 2024-11-15T23:59:10,505 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d24673a9e7414b9328c0b127e77893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731715147092 2024-11-15T23:59:10,505 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting edd907637f5b4721b4454b8e088a233f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731715147754 2024-11-15T23:59:10,506 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 02d70f0b6bef4c0ab88f4e0106d4e196, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148543 2024-11-15T23:59:10,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741974_1150 (size=57033) 2024-11-15T23:59:10,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3d7e826ecb654ee8aa4b544e7f08a04a 2024-11-15T23:59:10,532 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#123 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:10,535 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/1864278d39f749ce80aa06457f589d4c is 50, key is test_row_0/C:col10/1731715149184/Put/seqid=0 2024-11-15T23:59:10,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e0314ec514804d58aaed79f212a3ca9f is 50, key is test_row_0/B:col10/1731715150382/Put/seqid=0 2024-11-15T23:59:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741976_1152 (size=12151) 2024-11-15T23:59:10,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e0314ec514804d58aaed79f212a3ca9f 2024-11-15T23:59:10,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741975_1151 (size=12493) 2024-11-15T23:59:10,584 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/9bbcf32cc0224295b0782e0907439e1d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d 2024-11-15T23:59:10,593 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/1864278d39f749ce80aa06457f589d4c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1864278d39f749ce80aa06457f589d4c 2024-11-15T23:59:10,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/94ef83df2ef447e8be23c5872b237170 is 50, key is test_row_0/C:col10/1731715150382/Put/seqid=0 2024-11-15T23:59:10,598 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into 9bbcf32cc0224295b0782e0907439e1d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:10,598 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:10,598 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=12, startTime=1731715149940; duration=0sec 2024-11-15T23:59:10,603 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:10,603 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:10,605 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into 1864278d39f749ce80aa06457f589d4c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:10,605 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:10,605 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=12, startTime=1731715149940; duration=0sec 2024-11-15T23:59:10,605 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:10,605 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:10,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715210600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715210601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741977_1153 (size=12151) 2024-11-15T23:59:10,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715210610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715210611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715210760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,764 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715210820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715210820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715210830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715210830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:10,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T23:59:10,892 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:10,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T23:59:10,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:10,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:10,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:11,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/94ef83df2ef447e8be23c5872b237170 2024-11-15T23:59:11,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3d7e826ecb654ee8aa4b544e7f08a04a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a 2024-11-15T23:59:11,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a, entries=300, sequenceid=174, filesize=55.7 K 2024-11-15T23:59:11,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e0314ec514804d58aaed79f212a3ca9f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f 2024-11-15T23:59:11,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T23:59:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/94ef83df2ef447e8be23c5872b237170 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170 2024-11-15T23:59:11,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T23:59:11,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 583360a8f892c9e569e274bdbe35bf48 in 680ms, sequenceid=174, compaction requested=false 2024-11-15T23:59:11,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:11,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:11,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T23:59:11,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eae94b82948543afb70744d604f143e7_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715211152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715211157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715211159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715211159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741978_1154 (size=12304) 2024-11-15T23:59:11,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:11,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715211269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715211270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715211270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715211270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715211481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715211481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715211482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715211482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:11,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eae94b82948543afb70744d604f143e7_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eae94b82948543afb70744d604f143e7_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:11,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/7339394af85941ca84b22f305385f573, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:11,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/7339394af85941ca84b22f305385f573 is 175, key is test_row_0/A:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:11,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741979_1155 (size=31105) 2024-11-15T23:59:11,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715211791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715211792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715211800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715211800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:11,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:11,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/7339394af85941ca84b22f305385f573 2024-11-15T23:59:12,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/812aa34811144878be9ce041b9719009 is 50, key is test_row_0/B:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:12,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741980_1156 (size=12151) 2024-11-15T23:59:12,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:12,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715212301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:12,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715212302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715212320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:12,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715212321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:12,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/812aa34811144878be9ce041b9719009 2024-11-15T23:59:12,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e595f8bca5eb4c4ba0e366114a0c3330 is 50, key is test_row_0/C:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:12,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741981_1157 (size=12151) 2024-11-15T23:59:12,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e595f8bca5eb4c4ba0e366114a0c3330 2024-11-15T23:59:12,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/7339394af85941ca84b22f305385f573 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573 2024-11-15T23:59:12,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573, entries=150, sequenceid=203, filesize=30.4 K 2024-11-15T23:59:12,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/812aa34811144878be9ce041b9719009 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009 2024-11-15T23:59:12,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T23:59:12,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/e595f8bca5eb4c4ba0e366114a0c3330 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330 2024-11-15T23:59:12,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T23:59:12,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:12,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 583360a8f892c9e569e274bdbe35bf48 in 1482ms, sequenceid=203, compaction requested=true 2024-11-15T23:59:12,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:12,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:12,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:12,618 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:12,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:12,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:12,619 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:12,621 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:12,621 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:12,622 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,622 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/b2ba08546de24909b88baafff87f72c7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.9 K 2024-11-15T23:59:12,622 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:12,622 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:12,622 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,622 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=116.8 K 2024-11-15T23:59:12,622 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,622 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573] 2024-11-15T23:59:12,624 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ba08546de24909b88baafff87f72c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148543 2024-11-15T23:59:12,624 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bbcf32cc0224295b0782e0907439e1d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148543 2024-11-15T23:59:12,624 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e0314ec514804d58aaed79f212a3ca9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731715149191 2024-11-15T23:59:12,625 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 812aa34811144878be9ce041b9719009, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:12,625 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7e826ecb654ee8aa4b544e7f08a04a, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731715149191 2024-11-15T23:59:12,626 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7339394af85941ca84b22f305385f573, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:12,642 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:12,653 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:12,654 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/573f8287393040a69ef81ae6d0215b70 is 50, key is test_row_0/B:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:12,655 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111559c594e664da4105be7afcda554b689f_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:12,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111559c594e664da4105be7afcda554b689f_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:12,658 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111559c594e664da4105be7afcda554b689f_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741983_1159 (size=12595) 2024-11-15T23:59:12,694 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/573f8287393040a69ef81ae6d0215b70 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/573f8287393040a69ef81ae6d0215b70 2024-11-15T23:59:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741982_1158 (size=4469) 2024-11-15T23:59:12,706 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 573f8287393040a69ef81ae6d0215b70(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:12,706 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:12,706 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715152618; duration=0sec 2024-11-15T23:59:12,706 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:12,707 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:12,707 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:12,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:12,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:12,709 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,709 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1864278d39f749ce80aa06457f589d4c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=35.9 K 2024-11-15T23:59:12,711 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1864278d39f749ce80aa06457f589d4c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731715148543 2024-11-15T23:59:12,712 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ef83df2ef447e8be23c5872b237170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731715149191 2024-11-15T23:59:12,713 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e595f8bca5eb4c4ba0e366114a0c3330, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:12,724 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:12,724 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/fcc1a9a4256b4061a3e016a2d011b7bf is 50, key is test_row_0/C:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741984_1160 (size=12595) 2024-11-15T23:59:12,750 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/fcc1a9a4256b4061a3e016a2d011b7bf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fcc1a9a4256b4061a3e016a2d011b7bf 2024-11-15T23:59:12,759 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into fcc1a9a4256b4061a3e016a2d011b7bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:12,759 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:12,759 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715152618; duration=0sec 2024-11-15T23:59:12,759 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:12,759 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:12,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T23:59:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:12,771 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:12,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115083342a184fa4e7892302b4fafb64d45_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715151156/Put/seqid=0 2024-11-15T23:59:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:12,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:12,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741985_1161 (size=12304) 2024-11-15T23:59:12,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:12,808 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115083342a184fa4e7892302b4fafb64d45_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115083342a184fa4e7892302b4fafb64d45_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b039c4144dc642a3804cc0ba4e69875c, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b039c4144dc642a3804cc0ba4e69875c is 175, key is test_row_0/A:col10/1731715151156/Put/seqid=0 2024-11-15T23:59:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741986_1162 (size=31105) 2024-11-15T23:59:12,877 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b039c4144dc642a3804cc0ba4e69875c 2024-11-15T23:59:12,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/d24dda03ae924b7db59171c214d44d87 is 50, key is test_row_0/B:col10/1731715151156/Put/seqid=0 2024-11-15T23:59:12,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741987_1163 (size=12151) 2024-11-15T23:59:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715212954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715213070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,103 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:13,104 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/d3eb809b6077463386e21da43b5a8f1b is 175, key is test_row_0/A:col10/1731715151134/Put/seqid=0 2024-11-15T23:59:13,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741988_1164 (size=31549) 2024-11-15T23:59:13,145 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/d3eb809b6077463386e21da43b5a8f1b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b 2024-11-15T23:59:13,163 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into d3eb809b6077463386e21da43b5a8f1b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:13,163 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:13,163 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715152618; duration=0sec 2024-11-15T23:59:13,163 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:13,163 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:13,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715213289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715213320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715213320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,329 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,340 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/d24dda03ae924b7db59171c214d44d87 2024-11-15T23:59:13,328 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715213339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715213339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,344 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,344 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3d404b38dd054b51b70363c0dea4463d is 50, key is test_row_0/C:col10/1731715151156/Put/seqid=0 2024-11-15T23:59:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741989_1165 (size=12151) 2024-11-15T23:59:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715213601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:13,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:13,828 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3d404b38dd054b51b70363c0dea4463d 2024-11-15T23:59:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b039c4144dc642a3804cc0ba4e69875c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c 2024-11-15T23:59:13,845 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c, entries=150, sequenceid=216, filesize=30.4 K 2024-11-15T23:59:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/d24dda03ae924b7db59171c214d44d87 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87 2024-11-15T23:59:13,859 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T23:59:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3d404b38dd054b51b70363c0dea4463d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d 2024-11-15T23:59:13,870 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T23:59:13,874 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 583360a8f892c9e569e274bdbe35bf48 in 1103ms, sequenceid=216, compaction requested=false 2024-11-15T23:59:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T23:59:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T23:59:13,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T23:59:13,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9780 sec 2024-11-15T23:59:13,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9870 sec 2024-11-15T23:59:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:14,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:14,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T23:59:14,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151cc82549268348d68213f4e9666535d2_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:14,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741990_1166 (size=14794) 2024-11-15T23:59:14,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:14,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151cc82549268348d68213f4e9666535d2_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151cc82549268348d68213f4e9666535d2_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:14,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e77be32ee0a84d289cbc356feefdc75a, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:14,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e77be32ee0a84d289cbc356feefdc75a is 175, key is test_row_0/A:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715214192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:14,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741991_1167 (size=39749) 2024-11-15T23:59:14,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e77be32ee0a84d289cbc356feefdc75a 2024-11-15T23:59:14,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f82c680f874f4b2e93902aa94c3083f6 is 50, key is test_row_0/B:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:14,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741992_1168 (size=12151) 2024-11-15T23:59:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715214311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:14,472 INFO [master/0801813e30ae:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T23:59:14,472 INFO [master/0801813e30ae:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T23:59:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715214520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:14,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f82c680f874f4b2e93902aa94c3083f6 2024-11-15T23:59:14,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/a3cbc6420fd04bba8a1590c1820d0f0b is 50, key is test_row_0/C:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:14,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741993_1169 (size=12151) 2024-11-15T23:59:14,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715214830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T23:59:15,081 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:15,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T23:59:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:15,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:15,089 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:15,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:15,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/a3cbc6420fd04bba8a1590c1820d0f0b 2024-11-15T23:59:15,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e77be32ee0a84d289cbc356feefdc75a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a 2024-11-15T23:59:15,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a, entries=200, sequenceid=243, filesize=38.8 K 2024-11-15T23:59:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f82c680f874f4b2e93902aa94c3083f6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6 2024-11-15T23:59:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:15,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6, entries=150, sequenceid=243, filesize=11.9 K 2024-11-15T23:59:15,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/a3cbc6420fd04bba8a1590c1820d0f0b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b 2024-11-15T23:59:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b, entries=150, sequenceid=243, filesize=11.9 K 2024-11-15T23:59:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 583360a8f892c9e569e274bdbe35bf48 in 1081ms, sequenceid=243, compaction requested=true 2024-11-15T23:59:15,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:15,210 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:15,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,212 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:15,212 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:15,212 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:15,212 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=100.0 K 2024-11-15T23:59:15,212 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:15,212 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a] 2024-11-15T23:59:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,212 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:15,213 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:15,213 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:15,213 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/573f8287393040a69ef81ae6d0215b70, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.0 K 2024-11-15T23:59:15,213 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3eb809b6077463386e21da43b5a8f1b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,213 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 573f8287393040a69ef81ae6d0215b70, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:15,213 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b039c4144dc642a3804cc0ba4e69875c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731715151148 2024-11-15T23:59:15,214 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d24dda03ae924b7db59171c214d44d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731715151148 2024-11-15T23:59:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,214 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f82c680f874f4b2e93902aa94c3083f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:15,214 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77be32ee0a84d289cbc356feefdc75a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,236 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:15,237 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/073bbf50268f43bd952a853ad6c89ba9 is 50, key is test_row_0/B:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T23:59:15,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,243 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T23:59:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,246 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,257 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111540ea4a326cf24ff4bc070ba0a8723698_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,259 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111540ea4a326cf24ff4bc070ba0a8723698_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:15,260 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111540ea4a326cf24ff4bc070ba0a8723698_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741994_1170 (size=12697) 2024-11-15T23:59:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d862fd164c1043acbb440b9ab1a8f7a4_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715154145/Put/seqid=0 2024-11-15T23:59:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741995_1171 (size=4469) 2024-11-15T23:59:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741996_1172 (size=9814) 2024-11-15T23:59:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,318 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#139 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:15,319 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e6733559c2184c1f9b21e7d2af6ac071 is 175, key is test_row_0/A:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741997_1173 (size=31651) 2024-11-15T23:59:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:15,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715215430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715215433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715215434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715215438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715215438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715215540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715215559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715215559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715215559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715215560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,694 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/073bbf50268f43bd952a853ad6c89ba9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/073bbf50268f43bd952a853ad6c89ba9 2024-11-15T23:59:15,709 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 073bbf50268f43bd952a853ad6c89ba9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:15,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:15,709 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715155210; duration=0sec 2024-11-15T23:59:15,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:15,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:15,709 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:15,714 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:15,714 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:15,714 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:15,714 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fcc1a9a4256b4061a3e016a2d011b7bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.0 K 2024-11-15T23:59:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:15,715 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc1a9a4256b4061a3e016a2d011b7bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731715150486 2024-11-15T23:59:15,715 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d404b38dd054b51b70363c0dea4463d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731715151148 2024-11-15T23:59:15,716 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a3cbc6420fd04bba8a1590c1820d0f0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:15,720 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d862fd164c1043acbb440b9ab1a8f7a4_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d862fd164c1043acbb440b9ab1a8f7a4_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/946f00de19db4260acac19db409d73a7, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/946f00de19db4260acac19db409d73a7 is 175, key is test_row_0/A:col10/1731715154145/Put/seqid=0 2024-11-15T23:59:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715215749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,756 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:15,757 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/efa45083051548ac828356032ba2e869 is 50, key is test_row_0/C:col10/1731715152951/Put/seqid=0 2024-11-15T23:59:15,760 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/e6733559c2184c1f9b21e7d2af6ac071 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071 2024-11-15T23:59:15,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741998_1174 (size=22461) 2024-11-15T23:59:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,771 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into e6733559c2184c1f9b21e7d2af6ac071(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715215770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,771 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:15,771 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715155209; duration=0sec 2024-11-15T23:59:15,771 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:15,771 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715215770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715215770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715215779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:15,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741999_1175 (size=12697) 2024-11-15T23:59:15,818 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/efa45083051548ac828356032ba2e869 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/efa45083051548ac828356032ba2e869 2024-11-15T23:59:15,832 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into efa45083051548ac828356032ba2e869(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:15,832 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:15,832 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715155210; duration=0sec 2024-11-15T23:59:15,832 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:15,832 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715216061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715216089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715216089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715216089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715216089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,169 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/946f00de19db4260acac19db409d73a7 2024-11-15T23:59:16,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/471acd914b02494496efb2ba2404cbdc is 50, key is test_row_0/B:col10/1731715154145/Put/seqid=0 2024-11-15T23:59:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742000_1176 (size=9757) 2024-11-15T23:59:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715216580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715216600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715216600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715216600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715216600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:16,610 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/471acd914b02494496efb2ba2404cbdc 2024-11-15T23:59:16,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3b9a8af08ede493cb5dd634a3e3a3371 is 50, key is test_row_0/C:col10/1731715154145/Put/seqid=0 2024-11-15T23:59:16,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742001_1177 (size=9757) 2024-11-15T23:59:17,037 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3b9a8af08ede493cb5dd634a3e3a3371 2024-11-15T23:59:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/946f00de19db4260acac19db409d73a7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7 2024-11-15T23:59:17,056 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7, entries=100, sequenceid=253, filesize=21.9 K 2024-11-15T23:59:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/471acd914b02494496efb2ba2404cbdc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc 2024-11-15T23:59:17,062 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc, entries=100, sequenceid=253, filesize=9.5 K 2024-11-15T23:59:17,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/3b9a8af08ede493cb5dd634a3e3a3371 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371 2024-11-15T23:59:17,069 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371, entries=100, sequenceid=253, filesize=9.5 K 2024-11-15T23:59:17,070 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 583360a8f892c9e569e274bdbe35bf48 in 1827ms, sequenceid=253, compaction requested=false 2024-11-15T23:59:17,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:17,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:17,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T23:59:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T23:59:17,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T23:59:17,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-15T23:59:17,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9950 sec 2024-11-15T23:59:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T23:59:17,231 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:17,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T23:59:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:59:17,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:17,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:17,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:59:17,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T23:59:17,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:17,387 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T23:59:17,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115971c146106c742abb4eb1f7805b6d1dd_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715155429/Put/seqid=0 2024-11-15T23:59:17,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742002_1178 (size=12454) 2024-11-15T23:59:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:17,418 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115971c146106c742abb4eb1f7805b6d1dd_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115971c146106c742abb4eb1f7805b6d1dd_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/2d5b613fe03b45fdb084f17fcc8c8277, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:17,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/2d5b613fe03b45fdb084f17fcc8c8277 is 175, key is test_row_0/A:col10/1731715155429/Put/seqid=0 2024-11-15T23:59:17,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742003_1179 (size=31255) 2024-11-15T23:59:17,426 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/2d5b613fe03b45fdb084f17fcc8c8277 2024-11-15T23:59:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/df6c0a84079f417a8c5cd2533688446d is 50, key is test_row_0/B:col10/1731715155429/Put/seqid=0 2024-11-15T23:59:17,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742004_1180 (size=12301) 2024-11-15T23:59:17,469 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/df6c0a84079f417a8c5cd2533688446d 2024-11-15T23:59:17,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8040ef7b8b5645c5b9889dda662c40eb is 50, key is test_row_0/C:col10/1731715155429/Put/seqid=0 2024-11-15T23:59:17,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742005_1181 (size=12301) 2024-11-15T23:59:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:59:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:17,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:17,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715217610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715217613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715217620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715217620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,622 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715217622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,622 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,623 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715217720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715217725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:59:17,889 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8040ef7b8b5645c5b9889dda662c40eb 2024-11-15T23:59:17,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/2d5b613fe03b45fdb084f17fcc8c8277 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277 2024-11-15T23:59:17,905 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277, entries=150, sequenceid=283, filesize=30.5 K 2024-11-15T23:59:17,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/df6c0a84079f417a8c5cd2533688446d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d 2024-11-15T23:59:17,912 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T23:59:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8040ef7b8b5645c5b9889dda662c40eb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb 2024-11-15T23:59:17,923 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T23:59:17,924 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 583360a8f892c9e569e274bdbe35bf48 in 536ms, sequenceid=283, compaction requested=true 2024-11-15T23:59:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T23:59:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T23:59:17,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T23:59:17,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 692 msec 2024-11-15T23:59:17,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 698 msec 2024-11-15T23:59:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:17,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:17,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:17,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115068c4ce6b9154326adcfd9a8237e0d76_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:17,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742006_1182 (size=17534) 2024-11-15T23:59:17,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:18,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115068c4ce6b9154326adcfd9a8237e0d76_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068c4ce6b9154326adcfd9a8237e0d76_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:18,003 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/62bf077ad6f54267bcc75286d3427e30, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:18,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/62bf077ad6f54267bcc75286d3427e30 is 175, key is test_row_0/A:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742007_1183 (size=48639) 2024-11-15T23:59:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715218024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715218024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/62bf077ad6f54267bcc75286d3427e30 2024-11-15T23:59:18,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/8db9821cc55c4d18a8660a923a9e0456 is 50, key is test_row_0/B:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:18,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742008_1184 (size=12301) 2024-11-15T23:59:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715218140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715218140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715218350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715218350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T23:59:18,370 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:18,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T23:59:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:18,372 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:18,373 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:18,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:18,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/8db9821cc55c4d18a8660a923a9e0456 2024-11-15T23:59:18,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d5382c6569604af7b10fb55889d9a234 is 50, key is test_row_0/C:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:18,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742009_1185 (size=12301) 2024-11-15T23:59:18,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:59:18,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:18,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715218660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715218660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:18,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:59:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:18,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:59:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:18,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d5382c6569604af7b10fb55889d9a234 2024-11-15T23:59:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/62bf077ad6f54267bcc75286d3427e30 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30 2024-11-15T23:59:18,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30, entries=250, sequenceid=295, filesize=47.5 K 2024-11-15T23:59:18,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/8db9821cc55c4d18a8660a923a9e0456 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456 2024-11-15T23:59:18,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T23:59:18,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/d5382c6569604af7b10fb55889d9a234 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234 2024-11-15T23:59:18,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234, entries=150, sequenceid=295, filesize=12.0 K 2024-11-15T23:59:18,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 583360a8f892c9e569e274bdbe35bf48 in 992ms, sequenceid=295, compaction requested=true 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:18,926 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:18,926 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:18,928 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:18,928 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:18,928 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:18,928 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:18,928 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,928 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/073bbf50268f43bd952a853ad6c89ba9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=46.0 K 2024-11-15T23:59:18,929 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,929 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=130.9 K 2024-11-15T23:59:18,929 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,929 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30] 2024-11-15T23:59:18,930 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 073bbf50268f43bd952a853ad6c89ba9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:18,930 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6733559c2184c1f9b21e7d2af6ac071, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:18,930 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 471acd914b02494496efb2ba2404cbdc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731715154145 2024-11-15T23:59:18,930 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946f00de19db4260acac19db409d73a7, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731715154145 2024-11-15T23:59:18,931 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting df6c0a84079f417a8c5cd2533688446d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731715155427 2024-11-15T23:59:18,931 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5b613fe03b45fdb084f17fcc8c8277, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731715155427 2024-11-15T23:59:18,931 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db9821cc55c4d18a8660a923a9e0456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157596 2024-11-15T23:59:18,931 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62bf077ad6f54267bcc75286d3427e30, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157594 2024-11-15T23:59:18,960 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:18,961 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/10d164cc378342059eb46557df490335 is 50, key is test_row_0/B:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:18,970 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:18,994 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115336ab98ecf8c44d5be7e413ccf86791c_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:18,997 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115336ab98ecf8c44d5be7e413ccf86791c_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:18,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T23:59:18,997 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115336ab98ecf8c44d5be7e413ccf86791c_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:18,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:18,998 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742010_1186 (size=12983) 2024-11-15T23:59:19,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742011_1187 (size=4469) 2024-11-15T23:59:19,034 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#151 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:19,035 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/c513f2e001424348b840db9d5692ec56 is 175, key is test_row_0/A:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b7b01c80d1084ac0b5795561fae0a06b_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715158016/Put/seqid=0 2024-11-15T23:59:19,051 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/10d164cc378342059eb46557df490335 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/10d164cc378342059eb46557df490335 2024-11-15T23:59:19,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742012_1188 (size=31937) 2024-11-15T23:59:19,059 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 10d164cc378342059eb46557df490335(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:19,059 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:19,059 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=12, startTime=1731715158926; duration=0sec 2024-11-15T23:59:19,060 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:19,060 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:19,060 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:19,065 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:19,065 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:19,065 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:19,066 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/efa45083051548ac828356032ba2e869, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=46.0 K 2024-11-15T23:59:19,066 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting efa45083051548ac828356032ba2e869, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731715152875 2024-11-15T23:59:19,071 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9a8af08ede493cb5dd634a3e3a3371, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731715154145 2024-11-15T23:59:19,071 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/c513f2e001424348b840db9d5692ec56 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56 2024-11-15T23:59:19,072 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8040ef7b8b5645c5b9889dda662c40eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731715155427 2024-11-15T23:59:19,078 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d5382c6569604af7b10fb55889d9a234, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157596 2024-11-15T23:59:19,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742013_1189 (size=12454) 2024-11-15T23:59:19,083 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into c513f2e001424348b840db9d5692ec56(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:19,083 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:19,083 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=12, startTime=1731715158926; duration=0sec 2024-11-15T23:59:19,083 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:19,083 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:19,089 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b7b01c80d1084ac0b5795561fae0a06b_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b7b01c80d1084ac0b5795561fae0a06b_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:19,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3cd8788347e142679c28c7ae9e482c4f, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3cd8788347e142679c28c7ae9e482c4f is 175, key is test_row_0/A:col10/1731715158016/Put/seqid=0 2024-11-15T23:59:19,112 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:19,114 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c139243dec8044d5aeaaef863243875f is 50, key is test_row_0/C:col10/1731715157596/Put/seqid=0 2024-11-15T23:59:19,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742014_1190 (size=31255) 2024-11-15T23:59:19,123 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3cd8788347e142679c28c7ae9e482c4f 2024-11-15T23:59:19,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/099d015c8b0b4ab899db269578eed067 is 50, key is test_row_0/B:col10/1731715158016/Put/seqid=0 2024-11-15T23:59:19,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742015_1191 (size=12983) 2024-11-15T23:59:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742016_1192 (size=12301) 2024-11-15T23:59:19,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715219194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715219195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715219300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715219309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:19,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715219510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715219520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,573 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c139243dec8044d5aeaaef863243875f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c139243dec8044d5aeaaef863243875f 2024-11-15T23:59:19,582 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/099d015c8b0b4ab899db269578eed067 2024-11-15T23:59:19,590 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into c139243dec8044d5aeaaef863243875f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:19,590 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:19,590 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=12, startTime=1731715158926; duration=0sec 2024-11-15T23:59:19,590 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:19,590 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8d0c143c30934984a4ec65900a2832bf is 50, key is test_row_0/C:col10/1731715158016/Put/seqid=0 2024-11-15T23:59:19,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715219630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742017_1193 (size=12301) 2024-11-15T23:59:19,638 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715219640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,647 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715219640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,648 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715219830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715219830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,037 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8d0c143c30934984a4ec65900a2832bf 2024-11-15T23:59:20,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3cd8788347e142679c28c7ae9e482c4f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f 2024-11-15T23:59:20,077 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f, entries=150, sequenceid=319, filesize=30.5 K 2024-11-15T23:59:20,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/099d015c8b0b4ab899db269578eed067 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067 2024-11-15T23:59:20,093 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T23:59:20,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/8d0c143c30934984a4ec65900a2832bf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf 2024-11-15T23:59:20,112 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T23:59:20,119 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 583360a8f892c9e569e274bdbe35bf48 in 1121ms, sequenceid=319, compaction requested=false 2024-11-15T23:59:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T23:59:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T23:59:20,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T23:59:20,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7500 sec 2024-11-15T23:59:20,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7570 sec 2024-11-15T23:59:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:20,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:20,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:59:20,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115443f8d79903244c4af8284ba1ea7ac9e_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:20,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742018_1194 (size=17534) 2024-11-15T23:59:20,397 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:20,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115443f8d79903244c4af8284ba1ea7ac9e_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115443f8d79903244c4af8284ba1ea7ac9e_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:20,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/540a287cda964dd596c09a585e03e507, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:20,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/540a287cda964dd596c09a585e03e507 is 175, key is test_row_0/A:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:20,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742019_1195 (size=48639) 2024-11-15T23:59:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715220416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715220421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T23:59:20,520 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:20,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T23:59:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:20,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:20,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:20,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715220531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715220531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:20,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715220739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715220739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:20,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/540a287cda964dd596c09a585e03e507 2024-11-15T23:59:20,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/7990a393d03a41078a55227cbf037aa5 is 50, key is test_row_0/B:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:20,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742020_1196 (size=12301) 2024-11-15T23:59:20,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:20,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:20,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:21,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715221059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715221059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:21,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:21,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/7990a393d03a41078a55227cbf037aa5 2024-11-15T23:59:21,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/f9b586ec8ceb45cd9ce6c6d1267f74ce is 50, key is test_row_0/C:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742021_1197 (size=12301) 2024-11-15T23:59:21,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715221570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:21,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715221570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:21,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:21,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/f9b586ec8ceb45cd9ce6c6d1267f74ce 2024-11-15T23:59:21,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/540a287cda964dd596c09a585e03e507 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507 2024-11-15T23:59:21,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507, entries=250, sequenceid=336, filesize=47.5 K 2024-11-15T23:59:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/7990a393d03a41078a55227cbf037aa5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5 2024-11-15T23:59:21,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T23:59:21,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/f9b586ec8ceb45cd9ce6c6d1267f74ce as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce 2024-11-15T23:59:21,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T23:59:21,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 583360a8f892c9e569e274bdbe35bf48 in 1330ms, sequenceid=336, compaction requested=true 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:21,684 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:21,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:21,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:21,685 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:21,686 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:21,686 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:21,686 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,686 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=109.2 K 2024-11-15T23:59:21,686 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:21,686 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,686 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:21,686 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507] 2024-11-15T23:59:21,686 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,686 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/10d164cc378342059eb46557df490335, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.7 K 2024-11-15T23:59:21,687 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c513f2e001424348b840db9d5692ec56, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157596 2024-11-15T23:59:21,687 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d164cc378342059eb46557df490335, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157596 2024-11-15T23:59:21,688 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cd8788347e142679c28c7ae9e482c4f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731715158016 2024-11-15T23:59:21,688 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 099d015c8b0b4ab899db269578eed067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731715158016 2024-11-15T23:59:21,688 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540a287cda964dd596c09a585e03e507, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159182 2024-11-15T23:59:21,688 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7990a393d03a41078a55227cbf037aa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159194 2024-11-15T23:59:21,696 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:21,697 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:21,698 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/57858994c0524f0c87961465c63c3afe is 50, key is test_row_0/B:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:21,699 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111552916aa0ad1a497c8628633e40dcf088_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:21,700 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111552916aa0ad1a497c8628633e40dcf088_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:21,700 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111552916aa0ad1a497c8628633e40dcf088_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742022_1198 (size=13085) 2024-11-15T23:59:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742023_1199 (size=4469) 2024-11-15T23:59:21,736 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#160 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:21,736 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/1c6984aeff104909ba097af6efdcff39 is 175, key is test_row_0/A:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:21,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742024_1200 (size=32039) 2024-11-15T23:59:21,757 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/1c6984aeff104909ba097af6efdcff39 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39 2024-11-15T23:59:21,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T23:59:21,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,761 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T23:59:21,763 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into 1c6984aeff104909ba097af6efdcff39(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:21,763 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:21,763 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715161684; duration=0sec 2024-11-15T23:59:21,763 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:21,763 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:21,764 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:21,766 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:21,766 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:21,766 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:21,766 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c139243dec8044d5aeaaef863243875f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.7 K 2024-11-15T23:59:21,767 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c139243dec8044d5aeaaef863243875f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731715157596 2024-11-15T23:59:21,768 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0c143c30934984a4ec65900a2832bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731715158016 2024-11-15T23:59:21,768 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9b586ec8ceb45cd9ce6c6d1267f74ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159194 2024-11-15T23:59:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d3f8fc070ad46cea0c9141ac4479ea8_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715160414/Put/seqid=0 2024-11-15T23:59:21,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742025_1201 (size=12454) 2024-11-15T23:59:21,781 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:21,781 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/77260b7710da44579d911b36c38b1a11 is 50, key is test_row_0/C:col10/1731715159194/Put/seqid=0 2024-11-15T23:59:21,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742026_1202 (size=13085) 2024-11-15T23:59:22,126 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/57858994c0524f0c87961465c63c3afe as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/57858994c0524f0c87961465c63c3afe 2024-11-15T23:59:22,133 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 57858994c0524f0c87961465c63c3afe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:22,133 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:22,133 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715161684; duration=0sec 2024-11-15T23:59:22,133 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:22,133 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:22,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:22,201 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d3f8fc070ad46cea0c9141ac4479ea8_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d3f8fc070ad46cea0c9141ac4479ea8_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:22,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c4590b2e99e45f1b60570bba4230bb8, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:22,203 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/77260b7710da44579d911b36c38b1a11 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/77260b7710da44579d911b36c38b1a11 2024-11-15T23:59:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c4590b2e99e45f1b60570bba4230bb8 is 175, key is test_row_0/A:col10/1731715160414/Put/seqid=0 2024-11-15T23:59:22,210 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into 77260b7710da44579d911b36c38b1a11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:22,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:22,210 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715161684; duration=0sec 2024-11-15T23:59:22,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:22,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:22,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742027_1203 (size=31255) 2024-11-15T23:59:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. as already flushing 2024-11-15T23:59:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715222605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715222607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:22,618 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c4590b2e99e45f1b60570bba4230bb8 2024-11-15T23:59:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ae1d7ecece42433b888912056710748f is 50, key is test_row_0/B:col10/1731715160414/Put/seqid=0 2024-11-15T23:59:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742028_1204 (size=12301) 2024-11-15T23:59:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:22,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715222719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715222719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715222930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715222930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,048 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ae1d7ecece42433b888912056710748f 2024-11-15T23:59:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c877b357fdb34c49b8944c54ed539a6b is 50, key is test_row_0/C:col10/1731715160414/Put/seqid=0 2024-11-15T23:59:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742029_1205 (size=12301) 2024-11-15T23:59:23,079 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c877b357fdb34c49b8944c54ed539a6b 2024-11-15T23:59:23,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5c4590b2e99e45f1b60570bba4230bb8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8 2024-11-15T23:59:23,090 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8, entries=150, sequenceid=358, filesize=30.5 K 2024-11-15T23:59:23,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/ae1d7ecece42433b888912056710748f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f 2024-11-15T23:59:23,095 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T23:59:23,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/c877b357fdb34c49b8944c54ed539a6b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b 2024-11-15T23:59:23,101 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T23:59:23,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 583360a8f892c9e569e274bdbe35bf48 in 1341ms, sequenceid=358, compaction requested=false 2024-11-15T23:59:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:23,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T23:59:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T23:59:23,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T23:59:23,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5760 sec 2024-11-15T23:59:23,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5870 sec 2024-11-15T23:59:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:23,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:23,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T23:59:23,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d237eb9564e346ddaed9d26666d34771_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742030_1206 (size=14994) 2024-11-15T23:59:23,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:23,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d237eb9564e346ddaed9d26666d34771_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d237eb9564e346ddaed9d26666d34771_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:23,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b3590e1767c3494795c141f4b2f07c46, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b3590e1767c3494795c141f4b2f07c46 is 175, key is test_row_0/A:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715223291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742031_1207 (size=39949) 2024-11-15T23:59:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b3590e1767c3494795c141f4b2f07c46 2024-11-15T23:59:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715223293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/81281e800e7e413a8bba6248cbb6e697 is 50, key is test_row_0/B:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742032_1208 (size=12301) 2024-11-15T23:59:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715223400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715223401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715223610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715223610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40324 deadline: 1731715223659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,667 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40318 deadline: 1731715223669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,672 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40316 deadline: 1731715223680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,682 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in 583360a8f892c9e569e274bdbe35bf48 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/81281e800e7e413a8bba6248cbb6e697 2024-11-15T23:59:23,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/1d969a2f9616486b83e0000a6eb9fb94 is 50, key is test_row_0/C:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742033_1209 (size=12301) 2024-11-15T23:59:23,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/1d969a2f9616486b83e0000a6eb9fb94 2024-11-15T23:59:23,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/b3590e1767c3494795c141f4b2f07c46 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46 2024-11-15T23:59:23,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46, entries=200, sequenceid=376, filesize=39.0 K 2024-11-15T23:59:23,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/81281e800e7e413a8bba6248cbb6e697 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697 2024-11-15T23:59:23,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T23:59:23,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/1d969a2f9616486b83e0000a6eb9fb94 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94 2024-11-15T23:59:23,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T23:59:23,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 583360a8f892c9e569e274bdbe35bf48 in 512ms, sequenceid=376, compaction requested=true 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:23,754 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:23,754 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:23,756 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:23,756 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:23,756 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:23,756 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:23,756 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:23,756 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:23,756 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=100.8 K 2024-11-15T23:59:23,756 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/57858994c0524f0c87961465c63c3afe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.8 K 2024-11-15T23:59:23,756 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:23,756 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46] 2024-11-15T23:59:23,757 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 57858994c0524f0c87961465c63c3afe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159194 2024-11-15T23:59:23,757 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6984aeff104909ba097af6efdcff39, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159194 2024-11-15T23:59:23,758 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1d7ecece42433b888912056710748f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731715160400 2024-11-15T23:59:23,758 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4590b2e99e45f1b60570bba4230bb8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731715160400 2024-11-15T23:59:23,758 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 81281e800e7e413a8bba6248cbb6e697, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:23,758 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3590e1767c3494795c141f4b2f07c46, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:23,768 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,769 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:23,770 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/47d23163ef164f388cc0062109a8991a is 50, key is test_row_0/B:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,770 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150ef57226a4a840338bf4a929338163ff_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,772 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150ef57226a4a840338bf4a929338163ff_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,772 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150ef57226a4a840338bf4a929338163ff_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742035_1211 (size=4469) 2024-11-15T23:59:23,794 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#169 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:23,795 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/c803c2d62a0b46c3a440766c3dd82127 is 175, key is test_row_0/A:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742034_1210 (size=13187) 2024-11-15T23:59:23,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742036_1212 (size=32141) 2024-11-15T23:59:23,821 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/c803c2d62a0b46c3a440766c3dd82127 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127 2024-11-15T23:59:23,834 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into c803c2d62a0b46c3a440766c3dd82127(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:23,834 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:23,834 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715163754; duration=0sec 2024-11-15T23:59:23,834 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:23,834 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:23,834 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:23,835 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:23,835 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:23,835 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:23,836 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/77260b7710da44579d911b36c38b1a11, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.8 K 2024-11-15T23:59:23,836 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77260b7710da44579d911b36c38b1a11, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731715159194 2024-11-15T23:59:23,837 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c877b357fdb34c49b8944c54ed539a6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731715160400 2024-11-15T23:59:23,837 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d969a2f9616486b83e0000a6eb9fb94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:23,847 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:23,848 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/32bac28ab80948d2ab902f3a945690d4 is 50, key is test_row_0/C:col10/1731715162605/Put/seqid=0 2024-11-15T23:59:23,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742037_1213 (size=13187) 2024-11-15T23:59:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:23,923 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T23:59:23,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157fa7c493135b4dd9aaebb2399872236d_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715163290/Put/seqid=0 2024-11-15T23:59:23,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742038_1214 (size=12454) 2024-11-15T23:59:23,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:23,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157fa7c493135b4dd9aaebb2399872236d_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fa7c493135b4dd9aaebb2399872236d_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:23,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5490e3b272974a65aafd135cb6e6f745, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:23,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5490e3b272974a65aafd135cb6e6f745 is 175, key is test_row_0/A:col10/1731715163290/Put/seqid=0 2024-11-15T23:59:23,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715223953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715223954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:23,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742039_1215 (size=31255) 2024-11-15T23:59:23,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5490e3b272974a65aafd135cb6e6f745 2024-11-15T23:59:23,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/af60abaa2e6c4c149bd8c245d54abfe9 is 50, key is test_row_0/B:col10/1731715163290/Put/seqid=0 2024-11-15T23:59:24,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742040_1216 (size=12301) 2024-11-15T23:59:24,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/af60abaa2e6c4c149bd8c245d54abfe9 2024-11-15T23:59:24,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/47e8de9ba4754fdc9cffadf530ba2282 is 50, key is test_row_0/C:col10/1731715163290/Put/seqid=0 2024-11-15T23:59:24,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742041_1217 (size=12301) 2024-11-15T23:59:24,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715224070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:24,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715224070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:24,208 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/47d23163ef164f388cc0062109a8991a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/47d23163ef164f388cc0062109a8991a 2024-11-15T23:59:24,215 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 47d23163ef164f388cc0062109a8991a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:24,215 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:24,215 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715163754; duration=0sec 2024-11-15T23:59:24,215 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:24,215 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:24,261 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/32bac28ab80948d2ab902f3a945690d4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/32bac28ab80948d2ab902f3a945690d4 2024-11-15T23:59:24,266 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into 32bac28ab80948d2ab902f3a945690d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:24,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:24,266 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715163754; duration=0sec 2024-11-15T23:59:24,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:24,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40326 deadline: 1731715224280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40256 deadline: 1731715224280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-15T23:59:24,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/47e8de9ba4754fdc9cffadf530ba2282 2024-11-15T23:59:24,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/5490e3b272974a65aafd135cb6e6f745 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745 2024-11-15T23:59:24,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745, entries=150, sequenceid=400, filesize=30.5 K 2024-11-15T23:59:24,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/af60abaa2e6c4c149bd8c245d54abfe9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9 2024-11-15T23:59:24,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T23:59:24,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/47e8de9ba4754fdc9cffadf530ba2282 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282 2024-11-15T23:59:24,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T23:59:24,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 583360a8f892c9e569e274bdbe35bf48 in 528ms, sequenceid=400, compaction requested=false 2024-11-15T23:59:24,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:24,549 INFO [Thread-567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-567. 2024-11-15T23:59:24,549 DEBUG [Thread-567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,549 INFO [Thread-561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-561. 2024-11-15T23:59:24,549 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,549 DEBUG [Thread-561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,549 DEBUG [Thread-561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,549 DEBUG [Thread-561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,549 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,550 INFO [Thread-563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-563. 2024-11-15T23:59:24,550 DEBUG [Thread-563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,550 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,550 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,553 INFO [Thread-565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-565. 2024-11-15T23:59:24,553 DEBUG [Thread-565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,553 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,553 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:24,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T23:59:24,591 INFO [Thread-558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-558. 2024-11-15T23:59:24,591 DEBUG [Thread-558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,591 DEBUG [Thread-558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,591 DEBUG [Thread-558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,591 INFO [Thread-554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-554. 2024-11-15T23:59:24,591 DEBUG [Thread-554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:24,591 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,591 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:24,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:24,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115accdf5863f1f44c6a84ed6b50ea3cecb_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_0/A:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742042_1218 (size=12454) 2024-11-15T23:59:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T23:59:24,680 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:25,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:25,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115accdf5863f1f44c6a84ed6b50ea3cecb_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdf5863f1f44c6a84ed6b50ea3cecb_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:25,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/fbc1439236884c3288e792b08ccfae95, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:25,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/fbc1439236884c3288e792b08ccfae95 is 175, key is test_row_0/A:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742043_1219 (size=31255) 2024-11-15T23:59:25,412 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/fbc1439236884c3288e792b08ccfae95 2024-11-15T23:59:25,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e80a2999e1f44c839191e3493b410f62 is 50, key is test_row_0/B:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742044_1220 (size=12301) 2024-11-15T23:59:25,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e80a2999e1f44c839191e3493b410f62 2024-11-15T23:59:25,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/9037ab66573045c9bec99cfbceed11f8 is 50, key is test_row_0/C:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742045_1221 (size=12301) 2024-11-15T23:59:26,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/9037ab66573045c9bec99cfbceed11f8 2024-11-15T23:59:26,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/fbc1439236884c3288e792b08ccfae95 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95 2024-11-15T23:59:26,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95, entries=150, sequenceid=417, filesize=30.5 K 2024-11-15T23:59:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/e80a2999e1f44c839191e3493b410f62 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62 2024-11-15T23:59:26,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T23:59:26,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/9037ab66573045c9bec99cfbceed11f8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8 2024-11-15T23:59:26,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T23:59:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 583360a8f892c9e569e274bdbe35bf48 in 1684ms, sequenceid=417, compaction requested=true 2024-11-15T23:59:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 583360a8f892c9e569e274bdbe35bf48:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:26,276 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:26,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:26,276 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/A is initiating minor compaction (all files) 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/B is initiating minor compaction (all files) 2024-11-15T23:59:26,277 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/A in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:26,277 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/B in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:26,277 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=92.4 K 2024-11-15T23:59:26,277 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:26,277 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/47d23163ef164f388cc0062109a8991a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.9 K 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95] 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d23163ef164f388cc0062109a8991a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:26,277 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c803c2d62a0b46c3a440766c3dd82127, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:26,278 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting af60abaa2e6c4c149bd8c245d54abfe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731715163290 2024-11-15T23:59:26,278 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5490e3b272974a65aafd135cb6e6f745, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731715163290 2024-11-15T23:59:26,278 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e80a2999e1f44c839191e3493b410f62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731715163938 2024-11-15T23:59:26,278 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc1439236884c3288e792b08ccfae95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731715163938 2024-11-15T23:59:26,284 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:26,285 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:26,286 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/3688fdb55f5e4062bc85d849fa23a66d is 50, key is test_row_0/B:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:26,286 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155bd7a149597e426886fda4dd9e48cb3f_583360a8f892c9e569e274bdbe35bf48 store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742046_1222 (size=13289) 2024-11-15T23:59:26,304 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155bd7a149597e426886fda4dd9e48cb3f_583360a8f892c9e569e274bdbe35bf48, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:26,304 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155bd7a149597e426886fda4dd9e48cb3f_583360a8f892c9e569e274bdbe35bf48 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:26,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742047_1223 (size=4469) 2024-11-15T23:59:26,695 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/3688fdb55f5e4062bc85d849fa23a66d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/3688fdb55f5e4062bc85d849fa23a66d 2024-11-15T23:59:26,700 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/B of 583360a8f892c9e569e274bdbe35bf48 into 3688fdb55f5e4062bc85d849fa23a66d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:26,700 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:26,700 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/B, priority=13, startTime=1731715166275; duration=0sec 2024-11-15T23:59:26,700 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:26,700 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:B 2024-11-15T23:59:26,700 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:26,702 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:26,702 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 583360a8f892c9e569e274bdbe35bf48/C is initiating minor compaction (all files) 2024-11-15T23:59:26,702 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 583360a8f892c9e569e274bdbe35bf48/C in TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:26,702 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/32bac28ab80948d2ab902f3a945690d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp, totalSize=36.9 K 2024-11-15T23:59:26,702 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bac28ab80948d2ab902f3a945690d4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731715162587 2024-11-15T23:59:26,702 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e8de9ba4754fdc9cffadf530ba2282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731715163290 2024-11-15T23:59:26,703 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9037ab66573045c9bec99cfbceed11f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731715163938 2024-11-15T23:59:26,709 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:26,709 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 583360a8f892c9e569e274bdbe35bf48#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:26,710 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/fe1bb299e6fd474487ccbf475bc01ca3 is 50, key is test_row_0/C:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:26,710 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3fa81b4e6e354831864e4221179a3bd6 is 175, key is test_row_0/A:col10/1731715164589/Put/seqid=0 2024-11-15T23:59:26,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742049_1225 (size=32243) 2024-11-15T23:59:26,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742048_1224 (size=13289) 2024-11-15T23:59:27,119 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/3fa81b4e6e354831864e4221179a3bd6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3fa81b4e6e354831864e4221179a3bd6 2024-11-15T23:59:27,119 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/fe1bb299e6fd474487ccbf475bc01ca3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fe1bb299e6fd474487ccbf475bc01ca3 2024-11-15T23:59:27,124 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/C of 583360a8f892c9e569e274bdbe35bf48 into fe1bb299e6fd474487ccbf475bc01ca3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:27,124 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 583360a8f892c9e569e274bdbe35bf48/A of 583360a8f892c9e569e274bdbe35bf48 into 3fa81b4e6e354831864e4221179a3bd6(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 583360a8f892c9e569e274bdbe35bf48: 2024-11-15T23:59:27,124 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/C, priority=13, startTime=1731715166276; duration=0sec 2024-11-15T23:59:27,124 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., storeName=583360a8f892c9e569e274bdbe35bf48/A, priority=13, startTime=1731715166275; duration=0sec 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:C 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:27,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 583360a8f892c9e569e274bdbe35bf48:A 2024-11-15T23:59:27,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:59:33,701 INFO [Thread-556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-556. 2024-11-15T23:59:33,701 DEBUG [Thread-556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:33,701 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,701 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:33,711 INFO [Thread-552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-552. 2024-11-15T23:59:33,711 DEBUG [Thread-552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:33,711 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,711 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:33,741 INFO [Thread-550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-550. 2024-11-15T23:59:33,742 DEBUG [Thread-550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:33,742 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,742 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2611 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2560 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3563 rows 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1194 2024-11-15T23:59:33,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3579 rows 2024-11-15T23:59:33,742 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T23:59:33,742 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T23:59:33,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:33,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:33,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T23:59:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:33,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715173746"}]},"ts":"1731715173746"} 2024-11-15T23:59:33,748 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T23:59:33,748 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T23:59:33,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T23:59:33,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, UNASSIGN}] 2024-11-15T23:59:33,751 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, UNASSIGN 2024-11-15T23:59:33,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:33,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, UNASSIGN because future has completed 2024-11-15T23:59:33,753 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T23:59:33,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:59:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:33,905 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:33,905 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T23:59:33,905 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 583360a8f892c9e569e274bdbe35bf48, disabling compactions & flushes 2024-11-15T23:59:33,905 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:33,905 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:33,905 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. after waiting 0 ms 2024-11-15T23:59:33,906 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:33,906 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 583360a8f892c9e569e274bdbe35bf48 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T23:59:33,914 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111549b12f908c764e478a1ce90217ea43c3_583360a8f892c9e569e274bdbe35bf48 is 50, key is test_row_1/A:col10/1731715173710/Put/seqid=0 2024-11-15T23:59:33,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742050_1226 (size=9914) 2024-11-15T23:59:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:34,319 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:34,324 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111549b12f908c764e478a1ce90217ea43c3_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549b12f908c764e478a1ce90217ea43c3_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:34,325 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/753bb4b1f1034f209babe3ea783d70df, store: [table=TestAcidGuarantees family=A region=583360a8f892c9e569e274bdbe35bf48] 2024-11-15T23:59:34,325 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/753bb4b1f1034f209babe3ea783d70df is 175, key is test_row_1/A:col10/1731715173710/Put/seqid=0 2024-11-15T23:59:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742051_1227 (size=22561) 2024-11-15T23:59:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:34,730 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/753bb4b1f1034f209babe3ea783d70df 2024-11-15T23:59:34,744 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f3a0591d311140f6a6d10cca2f9c7178 is 50, key is test_row_1/B:col10/1731715173710/Put/seqid=0 2024-11-15T23:59:34,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742052_1228 (size=9857) 2024-11-15T23:59:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:35,153 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f3a0591d311140f6a6d10cca2f9c7178 2024-11-15T23:59:35,161 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/5170a476936a45e8ad22ddd25dc93a4c is 50, key is test_row_1/C:col10/1731715173710/Put/seqid=0 2024-11-15T23:59:35,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742053_1229 (size=9857) 2024-11-15T23:59:35,565 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/5170a476936a45e8ad22ddd25dc93a4c 2024-11-15T23:59:35,571 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/A/753bb4b1f1034f209babe3ea783d70df as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/753bb4b1f1034f209babe3ea783d70df 2024-11-15T23:59:35,575 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/753bb4b1f1034f209babe3ea783d70df, entries=100, sequenceid=426, filesize=22.0 K 2024-11-15T23:59:35,576 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/B/f3a0591d311140f6a6d10cca2f9c7178 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f3a0591d311140f6a6d10cca2f9c7178 2024-11-15T23:59:35,580 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f3a0591d311140f6a6d10cca2f9c7178, entries=100, sequenceid=426, filesize=9.6 K 2024-11-15T23:59:35,580 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/.tmp/C/5170a476936a45e8ad22ddd25dc93a4c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/5170a476936a45e8ad22ddd25dc93a4c 2024-11-15T23:59:35,584 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/5170a476936a45e8ad22ddd25dc93a4c, entries=100, sequenceid=426, filesize=9.6 K 2024-11-15T23:59:35,585 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 583360a8f892c9e569e274bdbe35bf48 in 1679ms, sequenceid=426, compaction requested=false 2024-11-15T23:59:35,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95] to archive 2024-11-15T23:59:35,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:35,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/cef29feaa32248ca850d3d64b5667649 2024-11-15T23:59:35,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5febf44d30184f8893554be56d846b73 2024-11-15T23:59:35,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d4faf6ce1b14469798acede067f63591 2024-11-15T23:59:35,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/a97a360811b64c1e871e921cc4cdc68b 2024-11-15T23:59:35,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/90d42a69deaa44158e0e539770d59495 2024-11-15T23:59:35,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/25689089e92b4da39dd9fd20ddd5b27d 2024-11-15T23:59:35,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/0a453cb244164dc4902d1eae6a8c8794 2024-11-15T23:59:35,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/6a7c7fd306d24141ad832233f0890125 2024-11-15T23:59:35,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c90c56d11ca44a78296fe8e6e932d18 2024-11-15T23:59:35,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5d213e13f4c141e290ba512132cfda70 2024-11-15T23:59:35,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/9bbcf32cc0224295b0782e0907439e1d 2024-11-15T23:59:35,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3d7e826ecb654ee8aa4b544e7f08a04a 2024-11-15T23:59:35,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/d3eb809b6077463386e21da43b5a8f1b 2024-11-15T23:59:35,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/7339394af85941ca84b22f305385f573 2024-11-15T23:59:35,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b039c4144dc642a3804cc0ba4e69875c 2024-11-15T23:59:35,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e77be32ee0a84d289cbc356feefdc75a 2024-11-15T23:59:35,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/e6733559c2184c1f9b21e7d2af6ac071 2024-11-15T23:59:35,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/946f00de19db4260acac19db409d73a7 2024-11-15T23:59:35,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/2d5b613fe03b45fdb084f17fcc8c8277 2024-11-15T23:59:35,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/62bf077ad6f54267bcc75286d3427e30 2024-11-15T23:59:35,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c513f2e001424348b840db9d5692ec56 2024-11-15T23:59:35,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3cd8788347e142679c28c7ae9e482c4f 2024-11-15T23:59:35,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/540a287cda964dd596c09a585e03e507 2024-11-15T23:59:35,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/1c6984aeff104909ba097af6efdcff39 2024-11-15T23:59:35,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5c4590b2e99e45f1b60570bba4230bb8 2024-11-15T23:59:35,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/b3590e1767c3494795c141f4b2f07c46 2024-11-15T23:59:35,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/c803c2d62a0b46c3a440766c3dd82127 2024-11-15T23:59:35,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/5490e3b272974a65aafd135cb6e6f745 2024-11-15T23:59:35,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/fbc1439236884c3288e792b08ccfae95 2024-11-15T23:59:35,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/6bb01306fcda40119e5d67cba2ef706b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/11d4a633f3d748eea7403c9451686f11, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/b2ba08546de24909b88baafff87f72c7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/573f8287393040a69ef81ae6d0215b70, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/073bbf50268f43bd952a853ad6c89ba9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/10d164cc378342059eb46557df490335, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/57858994c0524f0c87961465c63c3afe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/47d23163ef164f388cc0062109a8991a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62] to archive 2024-11-15T23:59:35,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:35,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/da3461a0d1bc4005b9ab03289e90ecdf 2024-11-15T23:59:35,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/03cb711e999b459cadcdea3792af9304 2024-11-15T23:59:35,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/6bb01306fcda40119e5d67cba2ef706b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/6bb01306fcda40119e5d67cba2ef706b 2024-11-15T23:59:35,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ea2cef52186745568f90865a214d8599 2024-11-15T23:59:35,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/67481dee91b9439ba8d7cc35e79ce3fb 2024-11-15T23:59:35,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/11d4a633f3d748eea7403c9451686f11 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/11d4a633f3d748eea7403c9451686f11 2024-11-15T23:59:35,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/68e0182558b14b0ba6dcfb1ea26c647c 2024-11-15T23:59:35,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/1739e759bb3e41c8b7d0563a6100e492 2024-11-15T23:59:35,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/feb3a2a254bd431dba7c54f80cbe8b27 2024-11-15T23:59:35,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/b2ba08546de24909b88baafff87f72c7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/b2ba08546de24909b88baafff87f72c7 2024-11-15T23:59:35,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/32f6f180f96044b6a9363b44ff131984 2024-11-15T23:59:35,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e0314ec514804d58aaed79f212a3ca9f 2024-11-15T23:59:35,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/573f8287393040a69ef81ae6d0215b70 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/573f8287393040a69ef81ae6d0215b70 2024-11-15T23:59:35,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/812aa34811144878be9ce041b9719009 2024-11-15T23:59:35,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/d24dda03ae924b7db59171c214d44d87 2024-11-15T23:59:35,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/073bbf50268f43bd952a853ad6c89ba9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/073bbf50268f43bd952a853ad6c89ba9 2024-11-15T23:59:35,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f82c680f874f4b2e93902aa94c3083f6 2024-11-15T23:59:35,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/471acd914b02494496efb2ba2404cbdc 2024-11-15T23:59:35,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/df6c0a84079f417a8c5cd2533688446d 2024-11-15T23:59:35,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/10d164cc378342059eb46557df490335 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/10d164cc378342059eb46557df490335 2024-11-15T23:59:35,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/8db9821cc55c4d18a8660a923a9e0456 2024-11-15T23:59:35,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/099d015c8b0b4ab899db269578eed067 2024-11-15T23:59:35,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/57858994c0524f0c87961465c63c3afe to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/57858994c0524f0c87961465c63c3afe 2024-11-15T23:59:35,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/7990a393d03a41078a55227cbf037aa5 2024-11-15T23:59:35,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/ae1d7ecece42433b888912056710748f 2024-11-15T23:59:35,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/47d23163ef164f388cc0062109a8991a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/47d23163ef164f388cc0062109a8991a 2024-11-15T23:59:35,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/81281e800e7e413a8bba6248cbb6e697 2024-11-15T23:59:35,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/af60abaa2e6c4c149bd8c245d54abfe9 2024-11-15T23:59:35,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/e80a2999e1f44c839191e3493b410f62 2024-11-15T23:59:35,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8f5cae6fb0d24515bdd3840003dfd5e4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/ea92efceaf84487b95ccd704827c2c54, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1864278d39f749ce80aa06457f589d4c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fcc1a9a4256b4061a3e016a2d011b7bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/efa45083051548ac828356032ba2e869, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c139243dec8044d5aeaaef863243875f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/77260b7710da44579d911b36c38b1a11, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/32bac28ab80948d2ab902f3a945690d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8] to archive 2024-11-15T23:59:35,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T23:59:35,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d755ee1eb7cd42b58d7eb0062f9bf0e1 2024-11-15T23:59:35,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c78c67fc9d294c5d9b37b2c1b49c974a 2024-11-15T23:59:35,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8f5cae6fb0d24515bdd3840003dfd5e4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8f5cae6fb0d24515bdd3840003dfd5e4 2024-11-15T23:59:35,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e619a576d1e2451b8acd7795931e2f78 2024-11-15T23:59:35,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/492dd168911b42618a96a701af5d1381 2024-11-15T23:59:35,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/ea92efceaf84487b95ccd704827c2c54 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/ea92efceaf84487b95ccd704827c2c54 2024-11-15T23:59:35,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/03185af5aa174e80ba69c0aff6200a74 2024-11-15T23:59:35,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/08d24673a9e7414b9328c0b127e77893 2024-11-15T23:59:35,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/edd907637f5b4721b4454b8e088a233f 2024-11-15T23:59:35,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1864278d39f749ce80aa06457f589d4c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1864278d39f749ce80aa06457f589d4c 2024-11-15T23:59:35,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/02d70f0b6bef4c0ab88f4e0106d4e196 2024-11-15T23:59:35,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/94ef83df2ef447e8be23c5872b237170 2024-11-15T23:59:35,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fcc1a9a4256b4061a3e016a2d011b7bf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fcc1a9a4256b4061a3e016a2d011b7bf 2024-11-15T23:59:35,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/e595f8bca5eb4c4ba0e366114a0c3330 2024-11-15T23:59:35,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3d404b38dd054b51b70363c0dea4463d 2024-11-15T23:59:35,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/efa45083051548ac828356032ba2e869 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/efa45083051548ac828356032ba2e869 2024-11-15T23:59:35,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/a3cbc6420fd04bba8a1590c1820d0f0b 2024-11-15T23:59:35,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/3b9a8af08ede493cb5dd634a3e3a3371 2024-11-15T23:59:35,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8040ef7b8b5645c5b9889dda662c40eb 2024-11-15T23:59:35,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c139243dec8044d5aeaaef863243875f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c139243dec8044d5aeaaef863243875f 2024-11-15T23:59:35,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/d5382c6569604af7b10fb55889d9a234 2024-11-15T23:59:35,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/8d0c143c30934984a4ec65900a2832bf 2024-11-15T23:59:35,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/77260b7710da44579d911b36c38b1a11 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/77260b7710da44579d911b36c38b1a11 2024-11-15T23:59:35,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/f9b586ec8ceb45cd9ce6c6d1267f74ce 2024-11-15T23:59:35,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/c877b357fdb34c49b8944c54ed539a6b 2024-11-15T23:59:35,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/32bac28ab80948d2ab902f3a945690d4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/32bac28ab80948d2ab902f3a945690d4 2024-11-15T23:59:35,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/1d969a2f9616486b83e0000a6eb9fb94 2024-11-15T23:59:35,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/47e8de9ba4754fdc9cffadf530ba2282 2024-11-15T23:59:35,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/9037ab66573045c9bec99cfbceed11f8 2024-11-15T23:59:35,694 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=4 2024-11-15T23:59:35,695 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. 2024-11-15T23:59:35,695 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 583360a8f892c9e569e274bdbe35bf48: Waiting for close lock at 1731715173905Running coprocessor pre-close hooks at 1731715173905Disabling compacts and flushes for region at 1731715173905Disabling writes for close at 1731715173906 (+1 ms)Obtaining lock to block concurrent updates at 1731715173906Preparing flush snapshotting stores in 583360a8f892c9e569e274bdbe35bf48 at 1731715173906Finished memstore snapshotting TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731715173906Flushing stores of TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48. at 1731715173907 (+1 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/A: creating writer at 1731715173907Flushing mob file 583360a8f892c9e569e274bdbe35bf48/A: appending metadata at 1731715173913 (+6 ms)Flushing mob file 583360a8f892c9e569e274bdbe35bf48/A: closing flushed file at 1731715173913Flushing 583360a8f892c9e569e274bdbe35bf48/A: appending metadata at 1731715174325 (+412 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/A: closing flushed file at 1731715174325Flushing 583360a8f892c9e569e274bdbe35bf48/B: creating writer at 1731715174735 (+410 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/B: appending metadata at 1731715174743 (+8 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/B: closing flushed file at 1731715174744 (+1 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/C: creating writer at 1731715175158 (+414 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/C: appending metadata at 1731715175160 (+2 ms)Flushing 583360a8f892c9e569e274bdbe35bf48/C: closing flushed file at 1731715175160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531b50da: reopening flushed file at 1731715175570 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@486f5e2: reopening flushed file at 1731715175575 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd8464a: reopening flushed file at 1731715175580 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 583360a8f892c9e569e274bdbe35bf48 in 1679ms, sequenceid=426, compaction requested=false at 1731715175585 (+5 ms)Writing region close event to WAL at 1731715175691 (+106 ms)Running coprocessor post-close hooks at 1731715175695 (+4 ms)Closed at 1731715175695 2024-11-15T23:59:35,697 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,697 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=583360a8f892c9e569e274bdbe35bf48, regionState=CLOSED 2024-11-15T23:59:35,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:35,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T23:59:35,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 583360a8f892c9e569e274bdbe35bf48, server=0801813e30ae,43013,1731715110098 in 1.9460 sec 2024-11-15T23:59:35,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-15T23:59:35,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=583360a8f892c9e569e274bdbe35bf48, UNASSIGN in 1.9520 sec 2024-11-15T23:59:35,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T23:59:35,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-15T23:59:35,708 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715175707"}]},"ts":"1731715175707"} 2024-11-15T23:59:35,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T23:59:35,709 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T23:59:35,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-15T23:59:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T23:59:35,890 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:35,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T23:59:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,893 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:59:35,893 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits] 2024-11-15T23:59:35,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3fa81b4e6e354831864e4221179a3bd6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/3fa81b4e6e354831864e4221179a3bd6 2024-11-15T23:59:35,903 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/753bb4b1f1034f209babe3ea783d70df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/A/753bb4b1f1034f209babe3ea783d70df 2024-11-15T23:59:35,906 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/3688fdb55f5e4062bc85d849fa23a66d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/3688fdb55f5e4062bc85d849fa23a66d 2024-11-15T23:59:35,907 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f3a0591d311140f6a6d10cca2f9c7178 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/B/f3a0591d311140f6a6d10cca2f9c7178 2024-11-15T23:59:35,910 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/5170a476936a45e8ad22ddd25dc93a4c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/5170a476936a45e8ad22ddd25dc93a4c 2024-11-15T23:59:35,911 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fe1bb299e6fd474487ccbf475bc01ca3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/C/fe1bb299e6fd474487ccbf475bc01ca3 2024-11-15T23:59:35,915 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits/429.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48/recovered.edits/429.seqid 2024-11-15T23:59:35,915 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,915 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T23:59:35,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:59:35,917 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T23:59:35,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068c4ce6b9154326adcfd9a8237e0d76_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068c4ce6b9154326adcfd9a8237e0d76_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115083342a184fa4e7892302b4fafb64d45_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115083342a184fa4e7892302b4fafb64d45_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,925 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d3f8fc070ad46cea0c9141ac4479ea8_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d3f8fc070ad46cea0c9141ac4479ea8_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,926 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d4943868f65471ca9e40426ec0f6f76_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d4943868f65471ca9e40426ec0f6f76_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,927 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151cc82549268348d68213f4e9666535d2_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151cc82549268348d68213f4e9666535d2_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,929 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115443f8d79903244c4af8284ba1ea7ac9e_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115443f8d79903244c4af8284ba1ea7ac9e_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,930 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154541372ba2ae4879b42064ca4fccfc7e_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154541372ba2ae4879b42064ca4fccfc7e_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,932 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549b12f908c764e478a1ce90217ea43c3_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549b12f908c764e478a1ce90217ea43c3_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,933 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fa7c493135b4dd9aaebb2399872236d_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fa7c493135b4dd9aaebb2399872236d_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,937 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115971c146106c742abb4eb1f7805b6d1dd_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115971c146106c742abb4eb1f7805b6d1dd_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,939 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4413324c7942589ed02fc194382ce2_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4413324c7942589ed02fc194382ce2_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,941 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a36fe619c8134ec0aa5fb97706cc97ee_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a36fe619c8134ec0aa5fb97706cc97ee_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,943 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdf5863f1f44c6a84ed6b50ea3cecb_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdf5863f1f44c6a84ed6b50ea3cecb_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4f90509c07a44e49dd02530bad11b5f_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4f90509c07a44e49dd02530bad11b5f_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,962 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b7b01c80d1084ac0b5795561fae0a06b_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b7b01c80d1084ac0b5795561fae0a06b_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,964 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc44fdb04e9b4c26a924a50857d97296_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc44fdb04e9b4c26a924a50857d97296_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,966 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d237eb9564e346ddaed9d26666d34771_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d237eb9564e346ddaed9d26666d34771_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,967 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d307fa397d67414eaa88b7b5e7d3d6b6_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d307fa397d67414eaa88b7b5e7d3d6b6_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,969 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d659a1178ef84a9e8e7cafe439368f57_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d659a1178ef84a9e8e7cafe439368f57_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,971 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d862fd164c1043acbb440b9ab1a8f7a4_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d862fd164c1043acbb440b9ab1a8f7a4_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,972 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eae94b82948543afb70744d604f143e7_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eae94b82948543afb70744d604f143e7_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,977 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe04cfcde57a46e491592e5107e87778_583360a8f892c9e569e274bdbe35bf48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe04cfcde57a46e491592e5107e87778_583360a8f892c9e569e274bdbe35bf48 2024-11-15T23:59:35,978 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T23:59:35,980 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,982 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T23:59:35,984 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T23:59:35,985 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,985 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T23:59:35,986 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715175985"}]},"ts":"9223372036854775807"} 2024-11-15T23:59:35,988 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T23:59:35,988 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 583360a8f892c9e569e274bdbe35bf48, NAME => 'TestAcidGuarantees,,1731715142356.583360a8f892c9e569e274bdbe35bf48.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T23:59:35,988 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T23:59:35,988 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715175988"}]},"ts":"9223372036854775807"} 2024-11-15T23:59:35,990 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T23:59:35,990 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:35,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-15T23:59:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T23:59:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T23:59:36,000 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:36,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237 (was 237), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=570 (was 600), ProcessCount=11 (was 11), AvailableMemoryMB=5033 (was 4864) - AvailableMemoryMB LEAK? - 2024-11-15T23:59:36,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=570, ProcessCount=11, AvailableMemoryMB=5033 2024-11-15T23:59:36,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T23:59:36,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T23:59:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T23:59:36,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T23:59:36,027 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:36,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-15T23:59:36,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T23:59:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:36,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742054_1230 (size=959) 2024-11-15T23:59:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:36,437 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-15T23:59:36,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742055_1231 (size=53) 2024-11-15T23:59:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6582600b266a5542c0d5a9612f296646, disabling compactions & flushes 2024-11-15T23:59:36,843 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. after waiting 0 ms 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:36,843 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:36,843 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6582600b266a5542c0d5a9612f296646: Waiting for close lock at 1731715176843Disabling compacts and flushes for region at 1731715176843Disabling writes for close at 1731715176843Writing region close event to WAL at 1731715176843Closed at 1731715176843 2024-11-15T23:59:36,845 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T23:59:36,845 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715176845"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715176845"}]},"ts":"1731715176845"} 2024-11-15T23:59:36,847 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T23:59:36,847 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T23:59:36,848 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715176847"}]},"ts":"1731715176847"} 2024-11-15T23:59:36,849 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T23:59:36,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, ASSIGN}] 2024-11-15T23:59:36,850 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, ASSIGN 2024-11-15T23:59:36,851 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-15T23:59:37,002 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6582600b266a5542c0d5a9612f296646, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:37,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, ASSIGN because future has completed 2024-11-15T23:59:37,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098}] 2024-11-15T23:59:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:37,160 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:37,161 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} 2024-11-15T23:59:37,161 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,161 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T23:59:37,161 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,161 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,163 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,164 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6582600b266a5542c0d5a9612f296646 columnFamilyName A 2024-11-15T23:59:37,164 DEBUG [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:37,165 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(327): Store=6582600b266a5542c0d5a9612f296646/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:37,165 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,170 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6582600b266a5542c0d5a9612f296646 columnFamilyName B 2024-11-15T23:59:37,170 DEBUG [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:37,170 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(327): Store=6582600b266a5542c0d5a9612f296646/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:37,170 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,171 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6582600b266a5542c0d5a9612f296646 columnFamilyName C 2024-11-15T23:59:37,172 DEBUG [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T23:59:37,172 INFO [StoreOpener-6582600b266a5542c0d5a9612f296646-1 {}] regionserver.HStore(327): Store=6582600b266a5542c0d5a9612f296646/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T23:59:37,172 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,173 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,173 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,174 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,174 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,175 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T23:59:37,176 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,178 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T23:59:37,178 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 6582600b266a5542c0d5a9612f296646; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62461797, jitterRate=-0.06924669444561005}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T23:59:37,178 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:37,179 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 6582600b266a5542c0d5a9612f296646: Running coprocessor pre-open hook at 1731715177161Writing region info on filesystem at 1731715177161Initializing all the Stores at 1731715177162 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715177162Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715177163 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715177163Cleaning up temporary data from old regions at 1731715177174 (+11 ms)Running coprocessor post-open hooks at 1731715177178 (+4 ms)Region opened successfully at 1731715177179 (+1 ms) 2024-11-15T23:59:37,180 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., pid=57, masterSystemTime=1731715177156 2024-11-15T23:59:37,182 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:37,182 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:37,182 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6582600b266a5542c0d5a9612f296646, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-15T23:59:37,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-15T23:59:37,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T23:59:37,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 in 180 msec 2024-11-15T23:59:37,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-15T23:59:37,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, ASSIGN in 337 msec 2024-11-15T23:59:37,188 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T23:59:37,189 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715177189"}]},"ts":"1731715177189"} 2024-11-15T23:59:37,190 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T23:59:37,191 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T23:59:37,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-15T23:59:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T23:59:38,170 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:38,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f2d9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae672f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,175 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f49276a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,178 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T23:59:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-15T23:59:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fc03a96 2024-11-15T23:59:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T23:59:38,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T23:59:38,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624230c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb1df5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf873a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57fa494a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,187 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd747d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567c2bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181830d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cd1fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10182a8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,195 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd1cac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6484e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447061ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f18222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5ba193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143946e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b12f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620171b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,206 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a4fc22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758a7b1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72502a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-15T23:59:38,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T23:59:38,211 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-15T23:59:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T23:59:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-15T23:59:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd7fff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-15T23:59:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T23:59:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:38,212 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T23:59:38,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc53243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:59:38,214 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,214 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,214 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,222 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,222 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,230 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,230 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,230 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,231 DEBUG [Thread-1085 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,231 DEBUG [Thread-1085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,233 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:38,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:59:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-15T23:59:38,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:38,242 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:38,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:38,256 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,259 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T23:59:38,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,261 DEBUG [Thread-1096 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-15T23:59:38,261 DEBUG [Thread-1096 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T23:59:38,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T23:59:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-15T23:59:38,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/3f55a073197c4bcfa132a40688893811 is 50, key is test_row_0/A:col10/1731715178230/Put/seqid=0 2024-11-15T23:59:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715238274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715238275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715238275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715238277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715238285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742056_1232 (size=12001) 2024-11-15T23:59:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:38,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715238390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715238390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715238399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715238399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715238400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715238600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715238600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715238609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715238609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715238610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/3f55a073197c4bcfa132a40688893811 2024-11-15T23:59:38,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c085481af8764ca7ba7608d226433d0b is 50, key is test_row_0/B:col10/1731715178230/Put/seqid=0 2024-11-15T23:59:38,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742057_1233 (size=12001) 2024-11-15T23:59:38,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:38,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715238910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715238910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715238920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715238921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715238921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c085481af8764ca7ba7608d226433d0b 2024-11-15T23:59:39,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e840511b125a4b429c54a56e9807b186 is 50, key is test_row_0/C:col10/1731715178230/Put/seqid=0 2024-11-15T23:59:39,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742058_1234 (size=12001) 2024-11-15T23:59:39,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e840511b125a4b429c54a56e9807b186 2024-11-15T23:59:39,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/3f55a073197c4bcfa132a40688893811 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811 2024-11-15T23:59:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T23:59:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c085481af8764ca7ba7608d226433d0b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b 2024-11-15T23:59:39,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T23:59:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:39,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e840511b125a4b429c54a56e9807b186 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186 2024-11-15T23:59:39,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T23:59:39,388 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T23:59:39,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6582600b266a5542c0d5a9612f296646 in 1164ms, sequenceid=15, compaction requested=false 2024-11-15T23:59:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:39,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:39,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:39,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715239439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715239440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9fe53a50e8c7449d96b0a5430de01235 is 50, key is test_row_0/A:col10/1731715179423/Put/seqid=0 2024-11-15T23:59:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715239442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715239444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715239450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742059_1235 (size=14341) 2024-11-15T23:59:39,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9fe53a50e8c7449d96b0a5430de01235 2024-11-15T23:59:39,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:39,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d6895795a298409e845bcf4cdd08ac38 is 50, key is test_row_0/B:col10/1731715179423/Put/seqid=0 2024-11-15T23:59:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742060_1236 (size=12001) 2024-11-15T23:59:39,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d6895795a298409e845bcf4cdd08ac38 2024-11-15T23:59:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715239550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715239559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/b50987d7691d4450b6930d607646bf2f is 50, key is test_row_0/C:col10/1731715179423/Put/seqid=0 2024-11-15T23:59:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715239560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715239560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742061_1237 (size=12001) 2024-11-15T23:59:39,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/b50987d7691d4450b6930d607646bf2f 2024-11-15T23:59:39,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9fe53a50e8c7449d96b0a5430de01235 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235 2024-11-15T23:59:39,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235, entries=200, sequenceid=39, filesize=14.0 K 2024-11-15T23:59:39,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d6895795a298409e845bcf4cdd08ac38 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38 2024-11-15T23:59:39,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T23:59:39,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/b50987d7691d4450b6930d607646bf2f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f 2024-11-15T23:59:39,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T23:59:39,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 188ms, sequenceid=39, compaction requested=false 2024-11-15T23:59:39,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:39,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T23:59:39,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,656 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:59:39,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/27878fc9914c4e3aaa2f4506bcdc6032 is 50, key is test_row_0/A:col10/1731715179443/Put/seqid=0 2024-11-15T23:59:39,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742062_1238 (size=12001) 2024-11-15T23:59:39,672 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/27878fc9914c4e3aaa2f4506bcdc6032 2024-11-15T23:59:39,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5b7d35117ddb435f9d1781cb6b4fe663 is 50, key is test_row_0/B:col10/1731715179443/Put/seqid=0 2024-11-15T23:59:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742063_1239 (size=12001) 2024-11-15T23:59:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5b7d35117ddb435f9d1781cb6b4fe663 2024-11-15T23:59:39,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/57b50759e3194a16944b4e23bf04d0c4 is 50, key is test_row_0/C:col10/1731715179443/Put/seqid=0 2024-11-15T23:59:39,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742064_1240 (size=12001) 2024-11-15T23:59:39,737 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/57b50759e3194a16944b4e23bf04d0c4 2024-11-15T23:59:39,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/27878fc9914c4e3aaa2f4506bcdc6032 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032 2024-11-15T23:59:39,750 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:59:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5b7d35117ddb435f9d1781cb6b4fe663 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663 2024-11-15T23:59:39,757 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:59:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/57b50759e3194a16944b4e23bf04d0c4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4 2024-11-15T23:59:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:39,765 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T23:59:39,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:39,766 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for 6582600b266a5542c0d5a9612f296646 in 110ms, sequenceid=51, compaction requested=true 2024-11-15T23:59:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:39,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T23:59:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T23:59:39,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T23:59:39,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-11-15T23:59:39,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.5380 sec 2024-11-15T23:59:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:39,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:39,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:59:39,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/485a21f77cf447b9872056b0f1f58a17 is 50, key is test_row_0/A:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:39,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742065_1241 (size=12001) 2024-11-15T23:59:39,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715239866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715239867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715239872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715239874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715239979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715239979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715239990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715239990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715240190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715240190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715240200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715240200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/485a21f77cf447b9872056b0f1f58a17 2024-11-15T23:59:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4750ee4c84e848cca5ecf3372eba7f4a is 50, key is test_row_0/B:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742066_1242 (size=12001) 2024-11-15T23:59:40,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4750ee4c84e848cca5ecf3372eba7f4a 2024-11-15T23:59:40,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a6ddae5aa3534744b58147b3e76be7ec is 50, key is test_row_0/C:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:40,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742067_1243 (size=12001) 2024-11-15T23:59:40,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a6ddae5aa3534744b58147b3e76be7ec 2024-11-15T23:59:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/485a21f77cf447b9872056b0f1f58a17 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17 2024-11-15T23:59:40,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17, entries=150, sequenceid=64, filesize=11.7 K 2024-11-15T23:59:40,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4750ee4c84e848cca5ecf3372eba7f4a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a 2024-11-15T23:59:40,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a, entries=150, sequenceid=64, filesize=11.7 K 2024-11-15T23:59:40,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a6ddae5aa3534744b58147b3e76be7ec as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec 2024-11-15T23:59:40,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec, entries=150, sequenceid=64, filesize=11.7 K 2024-11-15T23:59:40,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6582600b266a5542c0d5a9612f296646 in 517ms, sequenceid=64, compaction requested=true 2024-11-15T23:59:40,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:40,302 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:40,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:40,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:40,302 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:40,304 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:40,304 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:40,305 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,305 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=49.2 K 2024-11-15T23:59:40,305 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:40,305 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:40,305 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,305 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=46.9 K 2024-11-15T23:59:40,306 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c085481af8764ca7ba7608d226433d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715178228 2024-11-15T23:59:40,306 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f55a073197c4bcfa132a40688893811, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715178228 2024-11-15T23:59:40,307 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe53a50e8c7449d96b0a5430de01235, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731715178273 2024-11-15T23:59:40,307 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d6895795a298409e845bcf4cdd08ac38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731715178281 2024-11-15T23:59:40,310 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27878fc9914c4e3aaa2f4506bcdc6032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731715179433 2024-11-15T23:59:40,312 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485a21f77cf447b9872056b0f1f58a17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:40,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7d35117ddb435f9d1781cb6b4fe663, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731715179433 2024-11-15T23:59:40,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4750ee4c84e848cca5ecf3372eba7f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:40,332 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:40,333 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/9497992df53e482ca042864cf1a9f28d is 50, key is test_row_0/B:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:40,342 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:40,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/5b5413efd9c64502a66d003efb3f634d is 50, key is test_row_0/A:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:40,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742069_1245 (size=12139) 2024-11-15T23:59:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742068_1244 (size=12139) 2024-11-15T23:59:40,365 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/5b5413efd9c64502a66d003efb3f634d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/5b5413efd9c64502a66d003efb3f634d 2024-11-15T23:59:40,371 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 5b5413efd9c64502a66d003efb3f634d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:40,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:40,371 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=12, startTime=1731715180302; duration=0sec 2024-11-15T23:59:40,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:40,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:40,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:40,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:40,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:40,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=46.9 K 2024-11-15T23:59:40,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e840511b125a4b429c54a56e9807b186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715178228 2024-11-15T23:59:40,374 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50987d7691d4450b6930d607646bf2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731715178281 2024-11-15T23:59:40,375 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57b50759e3194a16944b4e23bf04d0c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731715179433 2024-11-15T23:59:40,375 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6ddae5aa3534744b58147b3e76be7ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T23:59:40,390 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:40,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-15T23:59:40,402 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#197 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:40,403 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/dbe2940ce4b7416f800694246a60a678 is 50, key is test_row_0/C:col10/1731715179780/Put/seqid=0 2024-11-15T23:59:40,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:40,404 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:40,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742070_1246 (size=12139) 2024-11-15T23:59:40,433 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/dbe2940ce4b7416f800694246a60a678 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/dbe2940ce4b7416f800694246a60a678 2024-11-15T23:59:40,442 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into dbe2940ce4b7416f800694246a60a678(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:40,442 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:40,442 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=12, startTime=1731715180303; duration=0sec 2024-11-15T23:59:40,442 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:40,442 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:40,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:40,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:40,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d5140636be314e07a3a6826766df297d is 50, key is test_row_0/A:col10/1731715179855/Put/seqid=0 2024-11-15T23:59:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715240510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715240510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715240511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715240518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742071_1247 (size=14341) 2024-11-15T23:59:40,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715240521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d5140636be314e07a3a6826766df297d 2024-11-15T23:59:40,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:40,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:40,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d2ab5deab18c422997d51350ba4c20c1 is 50, key is test_row_0/B:col10/1731715179855/Put/seqid=0 2024-11-15T23:59:40,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742072_1248 (size=12001) 2024-11-15T23:59:40,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d2ab5deab18c422997d51350ba4c20c1 2024-11-15T23:59:40,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715240630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715240630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715240639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/55fc2a92686f4c16a316493a304bd8d6 is 50, key is test_row_0/C:col10/1731715179855/Put/seqid=0 2024-11-15T23:59:40,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:40,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742073_1249 (size=12001) 2024-11-15T23:59:40,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/55fc2a92686f4c16a316493a304bd8d6 2024-11-15T23:59:40,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d5140636be314e07a3a6826766df297d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d 2024-11-15T23:59:40,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d, entries=200, sequenceid=90, filesize=14.0 K 2024-11-15T23:59:40,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d2ab5deab18c422997d51350ba4c20c1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1 2024-11-15T23:59:40,765 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/9497992df53e482ca042864cf1a9f28d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9497992df53e482ca042864cf1a9f28d 2024-11-15T23:59:40,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T23:59:40,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/55fc2a92686f4c16a316493a304bd8d6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6 2024-11-15T23:59:40,771 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 9497992df53e482ca042864cf1a9f28d(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:40,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:40,771 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=12, startTime=1731715180302; duration=0sec 2024-11-15T23:59:40,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:40,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:40,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T23:59:40,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 313ms, sequenceid=90, compaction requested=false 2024-11-15T23:59:40,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:40,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:40,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:59:40,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a70a1a1ab6424f06bceba4205ea33349 is 50, key is test_row_0/A:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742074_1250 (size=14341) 2024-11-15T23:59:40,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:40,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715240891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715240891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:40,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715240898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715241010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715241010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715241010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:41,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:41,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:41,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715241029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715241030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715241219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715241220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715241220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,233 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:59:41,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a70a1a1ab6424f06bceba4205ea33349 2024-11-15T23:59:41,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c28be5275f0f4f0f8ba75ef599ffe0e9 is 50, key is test_row_0/B:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:41,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742075_1251 (size=12001) 2024-11-15T23:59:41,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c28be5275f0f4f0f8ba75ef599ffe0e9 2024-11-15T23:59:41,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/073cc293d5e74d679b0f65880f0fc32b is 50, key is test_row_0/C:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:41,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742076_1252 (size=12001) 2024-11-15T23:59:41,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/073cc293d5e74d679b0f65880f0fc32b 2024-11-15T23:59:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:41,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a70a1a1ab6424f06bceba4205ea33349 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349 2024-11-15T23:59:41,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349, entries=200, sequenceid=106, filesize=14.0 K 2024-11-15T23:59:41,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c28be5275f0f4f0f8ba75ef599ffe0e9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9 2024-11-15T23:59:41,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9, entries=150, sequenceid=106, filesize=11.7 K 2024-11-15T23:59:41,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/073cc293d5e74d679b0f65880f0fc32b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b 2024-11-15T23:59:41,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b, entries=150, sequenceid=106, filesize=11.7 K 2024-11-15T23:59:41,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6582600b266a5542c0d5a9612f296646 in 525ms, sequenceid=106, compaction requested=true 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:41,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:41,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:41,371 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:41,372 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:41,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:41,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,373 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/5b5413efd9c64502a66d003efb3f634d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=39.9 K 2024-11-15T23:59:41,374 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:41,374 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:41,374 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,374 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9497992df53e482ca042864cf1a9f28d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.3 K 2024-11-15T23:59:41,374 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5413efd9c64502a66d003efb3f634d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:41,375 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9497992df53e482ca042864cf1a9f28d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:41,375 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5140636be314e07a3a6826766df297d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731715179855 2024-11-15T23:59:41,375 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ab5deab18c422997d51350ba4c20c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731715179855 2024-11-15T23:59:41,376 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a70a1a1ab6424f06bceba4205ea33349, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:41,376 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c28be5275f0f4f0f8ba75ef599ffe0e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:41,404 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:41,404 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/6ad52222f3444ed29c3e539b6637b55c is 50, key is test_row_0/A:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:41,415 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:41,416 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/76d8e68203a94ace84c4c87d4d465806 is 50, key is test_row_0/B:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742077_1253 (size=12241) 2024-11-15T23:59:41,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742078_1254 (size=12241) 2024-11-15T23:59:41,479 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/76d8e68203a94ace84c4c87d4d465806 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/76d8e68203a94ace84c4c87d4d465806 2024-11-15T23:59:41,486 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 76d8e68203a94ace84c4c87d4d465806(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:41,486 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:41,486 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715181371; duration=0sec 2024-11-15T23:59:41,486 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:41,486 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:41,486 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:41,488 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:41,488 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:41,488 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,488 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/dbe2940ce4b7416f800694246a60a678, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.3 K 2024-11-15T23:59:41,489 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe2940ce4b7416f800694246a60a678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731715179772 2024-11-15T23:59:41,489 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 55fc2a92686f4c16a316493a304bd8d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731715179855 2024-11-15T23:59:41,490 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 073cc293d5e74d679b0f65880f0fc32b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:41,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T23:59:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:41,494 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T23:59:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/b32619ed57524530bf6e85d4544c921b is 50, key is test_row_0/A:col10/1731715180889/Put/seqid=0 2024-11-15T23:59:41,523 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:41,524 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/20fb765d80f34c33b24ba9be9c47e704 is 50, key is test_row_0/C:col10/1731715180844/Put/seqid=0 2024-11-15T23:59:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:41,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742079_1255 (size=12001) 2024-11-15T23:59:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:41,545 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/b32619ed57524530bf6e85d4544c921b 2024-11-15T23:59:41,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/454d06edf9c5437eb600962c416308e2 is 50, key is test_row_0/B:col10/1731715180889/Put/seqid=0 2024-11-15T23:59:41,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715241568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715241570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742080_1256 (size=12241) 2024-11-15T23:59:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715241571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742081_1257 (size=12001) 2024-11-15T23:59:41,584 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/454d06edf9c5437eb600962c416308e2 2024-11-15T23:59:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/936d0641037d42ad9da33dd4ca6307e5 is 50, key is test_row_0/C:col10/1731715180889/Put/seqid=0 2024-11-15T23:59:41,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742082_1258 (size=12001) 2024-11-15T23:59:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715241680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715241680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715241689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,873 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/6ad52222f3444ed29c3e539b6637b55c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6ad52222f3444ed29c3e539b6637b55c 2024-11-15T23:59:41,879 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 6ad52222f3444ed29c3e539b6637b55c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:41,880 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:41,880 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715181371; duration=0sec 2024-11-15T23:59:41,880 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:41,881 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715241890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715241890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715241900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:41,988 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/20fb765d80f34c33b24ba9be9c47e704 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/20fb765d80f34c33b24ba9be9c47e704 2024-11-15T23:59:41,996 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 20fb765d80f34c33b24ba9be9c47e704(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:41,996 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:41,996 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715181371; duration=0sec 2024-11-15T23:59:41,996 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:41,996 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:42,003 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/936d0641037d42ad9da33dd4ca6307e5 2024-11-15T23:59:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/b32619ed57524530bf6e85d4544c921b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b 2024-11-15T23:59:42,015 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T23:59:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/454d06edf9c5437eb600962c416308e2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2 2024-11-15T23:59:42,022 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T23:59:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/936d0641037d42ad9da33dd4ca6307e5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5 2024-11-15T23:59:42,028 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T23:59:42,029 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6582600b266a5542c0d5a9612f296646 in 535ms, sequenceid=128, compaction requested=false 2024-11-15T23:59:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-15T23:59:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-15T23:59:42,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T23:59:42,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-11-15T23:59:42,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.6440 sec 2024-11-15T23:59:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:42,043 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:42,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T23:59:42,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/41e1a819f447440995f97c8f6b2b0a0d is 50, key is test_row_0/A:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742083_1259 (size=14541) 2024-11-15T23:59:42,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/41e1a819f447440995f97c8f6b2b0a0d 2024-11-15T23:59:42,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/58cb5f3af44e4806b2b557d6630dc917 is 50, key is test_row_0/B:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715242112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715242114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742084_1260 (size=12151) 2024-11-15T23:59:42,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/58cb5f3af44e4806b2b557d6630dc917 2024-11-15T23:59:42,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/829f167a024b479294191a85b3adfe00 is 50, key is test_row_0/C:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742085_1261 (size=12151) 2024-11-15T23:59:42,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/829f167a024b479294191a85b3adfe00 2024-11-15T23:59:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/41e1a819f447440995f97c8f6b2b0a0d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d 2024-11-15T23:59:42,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d, entries=200, sequenceid=146, filesize=14.2 K 2024-11-15T23:59:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715242199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715242200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/58cb5f3af44e4806b2b557d6630dc917 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917 2024-11-15T23:59:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715242211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715242220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715242220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917, entries=150, sequenceid=146, filesize=11.9 K 2024-11-15T23:59:42,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/829f167a024b479294191a85b3adfe00 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00 2024-11-15T23:59:42,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00, entries=150, sequenceid=146, filesize=11.9 K 2024-11-15T23:59:42,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6582600b266a5542c0d5a9612f296646 in 188ms, sequenceid=146, compaction requested=true 2024-11-15T23:59:42,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:42,232 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:42,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:59:42,232 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:42,233 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:42,233 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:42,233 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:42,233 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:42,233 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,233 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,233 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/76d8e68203a94ace84c4c87d4d465806, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.5 K 2024-11-15T23:59:42,233 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6ad52222f3444ed29c3e539b6637b55c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.9 K 2024-11-15T23:59:42,234 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad52222f3444ed29c3e539b6637b55c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:42,234 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d8e68203a94ace84c4c87d4d465806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:42,234 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 454d06edf9c5437eb600962c416308e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731715180889 2024-11-15T23:59:42,234 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b32619ed57524530bf6e85d4544c921b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731715180889 2024-11-15T23:59:42,235 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e1a819f447440995f97c8f6b2b0a0d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:42,235 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cb5f3af44e4806b2b557d6630dc917, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:42,251 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#213 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:42,251 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/da4ceb7e94fc42989286a8f075b8b135 is 50, key is test_row_0/B:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,256 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:42,257 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a6cea55cd66a4264807b72bd949b0421 is 50, key is test_row_0/A:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742086_1262 (size=12493) 2024-11-15T23:59:42,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742087_1263 (size=12493) 2024-11-15T23:59:42,333 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a6cea55cd66a4264807b72bd949b0421 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a6cea55cd66a4264807b72bd949b0421 2024-11-15T23:59:42,339 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into a6cea55cd66a4264807b72bd949b0421(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:42,339 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,339 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715182231; duration=0sec 2024-11-15T23:59:42,339 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:42,340 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:42,340 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:42,341 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:42,341 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:42,341 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,341 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/20fb765d80f34c33b24ba9be9c47e704, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.5 K 2024-11-15T23:59:42,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20fb765d80f34c33b24ba9be9c47e704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731715180519 2024-11-15T23:59:42,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936d0641037d42ad9da33dd4ca6307e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731715180889 2024-11-15T23:59:42,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829f167a024b479294191a85b3adfe00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:42,351 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:42,352 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/4c2888f8b0b54d708ab5770a538c96bf is 50, key is test_row_0/C:col10/1731715182041/Put/seqid=0 2024-11-15T23:59:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742088_1264 (size=12493) 2024-11-15T23:59:42,400 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/4c2888f8b0b54d708ab5770a538c96bf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/4c2888f8b0b54d708ab5770a538c96bf 2024-11-15T23:59:42,423 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 4c2888f8b0b54d708ab5770a538c96bf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:42,423 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,423 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715182232; duration=0sec 2024-11-15T23:59:42,423 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:42,423 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:42,433 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:42,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T23:59:42,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d762c903b938403b8a20022bbf7b39e3 is 50, key is test_row_0/A:col10/1731715182105/Put/seqid=0 2024-11-15T23:59:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742089_1265 (size=14541) 2024-11-15T23:59:42,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d762c903b938403b8a20022bbf7b39e3 2024-11-15T23:59:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715242469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715242469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f3edb806c64c4477870aad361b908eb0 is 50, key is test_row_0/B:col10/1731715182105/Put/seqid=0 2024-11-15T23:59:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742090_1266 (size=12151) 2024-11-15T23:59:42,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f3edb806c64c4477870aad361b908eb0 2024-11-15T23:59:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T23:59:42,556 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:42,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-15T23:59:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:42,567 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:42,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:42,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715242580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715242580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/43f842b751cb44d7af5b0284b60e6e57 is 50, key is test_row_0/C:col10/1731715182105/Put/seqid=0 2024-11-15T23:59:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742091_1267 (size=12151) 2024-11-15T23:59:42,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/43f842b751cb44d7af5b0284b60e6e57 2024-11-15T23:59:42,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d762c903b938403b8a20022bbf7b39e3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3 2024-11-15T23:59:42,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3, entries=200, sequenceid=170, filesize=14.2 K 2024-11-15T23:59:42,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f3edb806c64c4477870aad361b908eb0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0 2024-11-15T23:59:42,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T23:59:42,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/43f842b751cb44d7af5b0284b60e6e57 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57 2024-11-15T23:59:42,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T23:59:42,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 6582600b266a5542c0d5a9612f296646 in 233ms, sequenceid=170, compaction requested=false 2024-11-15T23:59:42,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:42,711 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/da4ceb7e94fc42989286a8f075b8b135 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/da4ceb7e94fc42989286a8f075b8b135 2024-11-15T23:59:42,717 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into da4ceb7e94fc42989286a8f075b8b135(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:42,718 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:42,718 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715182232; duration=0sec 2024-11-15T23:59:42,718 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:42,718 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:42,726 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:42,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T23:59:42,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:42,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/99a71dee221c4f51bfb1d7eec2f52cf3 is 50, key is test_row_0/A:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:42,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742092_1268 (size=14541) 2024-11-15T23:59:42,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/99a71dee221c4f51bfb1d7eec2f52cf3 2024-11-15T23:59:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715242783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715242784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715242788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715242790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715242790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/9d2b01199f344cabb406302864667e3a is 50, key is test_row_0/B:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:42,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742093_1269 (size=12151) 2024-11-15T23:59:42,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/9d2b01199f344cabb406302864667e3a 2024-11-15T23:59:42,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/311d8e2328ff4953b2a85d124f809d68 is 50, key is test_row_0/C:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:42,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:42,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:42,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:42,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715242900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715242900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715242900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:42,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742094_1270 (size=12151) 2024-11-15T23:59:43,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715243100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715243110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715243110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715243111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715243122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/311d8e2328ff4953b2a85d124f809d68 2024-11-15T23:59:43,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/99a71dee221c4f51bfb1d7eec2f52cf3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3 2024-11-15T23:59:43,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3, entries=200, sequenceid=187, filesize=14.2 K 2024-11-15T23:59:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/9d2b01199f344cabb406302864667e3a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a 2024-11-15T23:59:43,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a, entries=150, sequenceid=187, filesize=11.9 K 2024-11-15T23:59:43,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/311d8e2328ff4953b2a85d124f809d68 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68 2024-11-15T23:59:43,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68, entries=150, sequenceid=187, filesize=11.9 K 2024-11-15T23:59:43,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6582600b266a5542c0d5a9612f296646 in 658ms, sequenceid=187, compaction requested=true 2024-11-15T23:59:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:43,384 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:43,384 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:43,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:43,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:43,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:43,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:43,391 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:43,391 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:43,391 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,391 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a6cea55cd66a4264807b72bd949b0421, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=40.6 K 2024-11-15T23:59:43,392 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6cea55cd66a4264807b72bd949b0421, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:43,392 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d762c903b938403b8a20022bbf7b39e3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731715182103 2024-11-15T23:59:43,393 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99a71dee221c4f51bfb1d7eec2f52cf3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:43,393 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:43,393 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:43,393 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,393 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/da4ceb7e94fc42989286a8f075b8b135, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.9 K 2024-11-15T23:59:43,397 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting da4ceb7e94fc42989286a8f075b8b135, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:43,398 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f3edb806c64c4477870aad361b908eb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731715182103 2024-11-15T23:59:43,401 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2b01199f344cabb406302864667e3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:43,409 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:43,410 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/78c8a60a6c2e43c4a5a4aa854ca1c001 is 50, key is test_row_0/A:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:43,416 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:43,417 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/937f0194c7844e93a815534cc2287971 is 50, key is test_row_0/B:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:43,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:43,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T23:59:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:43,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/45d56a9ca2d546ea8666ca04f31aef77 is 50, key is test_row_0/A:col10/1731715183425/Put/seqid=0 2024-11-15T23:59:43,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742096_1272 (size=12595) 2024-11-15T23:59:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742095_1271 (size=12595) 2024-11-15T23:59:43,449 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/78c8a60a6c2e43c4a5a4aa854ca1c001 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/78c8a60a6c2e43c4a5a4aa854ca1c001 2024-11-15T23:59:43,449 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/937f0194c7844e93a815534cc2287971 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/937f0194c7844e93a815534cc2287971 2024-11-15T23:59:43,457 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 937f0194c7844e93a815534cc2287971(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:43,457 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:43,457 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715183384; duration=0sec 2024-11-15T23:59:43,457 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:43,457 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:43,457 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:43,458 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 78c8a60a6c2e43c4a5a4aa854ca1c001(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:43,458 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:43,458 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715183384; duration=0sec 2024-11-15T23:59:43,459 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:43,459 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:43,459 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:43,460 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:43,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742097_1273 (size=14541) 2024-11-15T23:59:43,460 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,460 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/4c2888f8b0b54d708ab5770a538c96bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=35.9 K 2024-11-15T23:59:43,461 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2888f8b0b54d708ab5770a538c96bf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731715181558 2024-11-15T23:59:43,462 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f842b751cb44d7af5b0284b60e6e57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731715182103 2024-11-15T23:59:43,462 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 311d8e2328ff4953b2a85d124f809d68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:43,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715243456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715243460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715243464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,473 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:43,474 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/f43b997de8c34dd48bae12fd372f02b9 is 50, key is test_row_0/C:col10/1731715182466/Put/seqid=0 2024-11-15T23:59:43,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742098_1274 (size=12595) 2024-11-15T23:59:43,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715243570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715243579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715243579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715243620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715243630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715243780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715243790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715243790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:43,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/45d56a9ca2d546ea8666ca04f31aef77 2024-11-15T23:59:43,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f69668ca57a648ba8e4d119256a74d79 is 50, key is test_row_0/B:col10/1731715183425/Put/seqid=0 2024-11-15T23:59:43,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742099_1275 (size=12151) 2024-11-15T23:59:43,888 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/f43b997de8c34dd48bae12fd372f02b9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/f43b997de8c34dd48bae12fd372f02b9 2024-11-15T23:59:43,894 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into f43b997de8c34dd48bae12fd372f02b9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:43,894 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:43,894 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715183385; duration=0sec 2024-11-15T23:59:43,895 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:43,895 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:43,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:43,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715244090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715244100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715244100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f69668ca57a648ba8e4d119256a74d79 2024-11-15T23:59:44,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/15302431f8aa412a82c2a7996d54189d is 50, key is test_row_0/C:col10/1731715183425/Put/seqid=0 2024-11-15T23:59:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742100_1276 (size=12151) 2024-11-15T23:59:44,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715244599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715244610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715244611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715244639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,642 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 6582600b266a5542c0d5a9612f296646 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715244650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:44,654 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_1' in 6582600b266a5542c0d5a9612f296646 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:44,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:44,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/15302431f8aa412a82c2a7996d54189d 2024-11-15T23:59:44,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/45d56a9ca2d546ea8666ca04f31aef77 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77 2024-11-15T23:59:44,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77, entries=200, sequenceid=209, filesize=14.2 K 2024-11-15T23:59:44,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f69668ca57a648ba8e4d119256a74d79 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79 2024-11-15T23:59:44,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:44,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T23:59:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/15302431f8aa412a82c2a7996d54189d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d 2024-11-15T23:59:44,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T23:59:44,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6582600b266a5542c0d5a9612f296646 in 1353ms, sequenceid=209, compaction requested=false 2024-11-15T23:59:44,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:44,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T23:59:44,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:44,906 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:59:44,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d1fb59c30ddd4de5bd929cafb8d0cc44 is 50, key is test_row_0/A:col10/1731715183462/Put/seqid=0 2024-11-15T23:59:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742101_1277 (size=12151) 2024-11-15T23:59:45,329 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d1fb59c30ddd4de5bd929cafb8d0cc44 2024-11-15T23:59:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/518e048e54244fb78575a291249fb470 is 50, key is test_row_0/B:col10/1731715183462/Put/seqid=0 2024-11-15T23:59:45,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742102_1278 (size=12151) 2024-11-15T23:59:45,426 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/518e048e54244fb78575a291249fb470 2024-11-15T23:59:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/18b6e4e7f5e841ad94d39d71659cb5e3 is 50, key is test_row_0/C:col10/1731715183462/Put/seqid=0 2024-11-15T23:59:45,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742103_1279 (size=12151) 2024-11-15T23:59:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:45,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715245664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715245666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715245671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715245780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715245780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715245780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:45,931 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/18b6e4e7f5e841ad94d39d71659cb5e3 2024-11-15T23:59:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d1fb59c30ddd4de5bd929cafb8d0cc44 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44 2024-11-15T23:59:45,948 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:59:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/518e048e54244fb78575a291249fb470 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470 2024-11-15T23:59:45,957 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:59:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/18b6e4e7f5e841ad94d39d71659cb5e3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3 2024-11-15T23:59:45,963 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3, entries=150, sequenceid=226, filesize=11.9 K 2024-11-15T23:59:45,965 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6582600b266a5542c0d5a9612f296646 in 1059ms, sequenceid=226, compaction requested=true 2024-11-15T23:59:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T23:59:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T23:59:45,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T23:59:45,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3970 sec 2024-11-15T23:59:45,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.4090 sec 2024-11-15T23:59:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:45,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:45,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T23:59:46,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/0689cb10b01d4c2fa86df29088433c3f is 50, key is test_row_0/A:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:46,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742104_1280 (size=12151) 2024-11-15T23:59:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715246020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715246026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715246031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715246130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715246140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715246140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715246340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715246351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715246365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/0689cb10b01d4c2fa86df29088433c3f 2024-11-15T23:59:46,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1236a651ab0f4dd0b2594c951a4616d3 is 50, key is test_row_0/B:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742105_1281 (size=12151) 2024-11-15T23:59:46,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1236a651ab0f4dd0b2594c951a4616d3 2024-11-15T23:59:46,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/486d0e4a57fb4e3eafc4bea0c4076a49 is 50, key is test_row_0/C:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:46,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742106_1282 (size=12151) 2024-11-15T23:59:46,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/486d0e4a57fb4e3eafc4bea0c4076a49 2024-11-15T23:59:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/0689cb10b01d4c2fa86df29088433c3f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f 2024-11-15T23:59:46,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:59:46,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1236a651ab0f4dd0b2594c951a4616d3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3 2024-11-15T23:59:46,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:59:46,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/486d0e4a57fb4e3eafc4bea0c4076a49 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49 2024-11-15T23:59:46,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49, entries=150, sequenceid=251, filesize=11.9 K 2024-11-15T23:59:46,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 651ms, sequenceid=251, compaction requested=true 2024-11-15T23:59:46,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T23:59:46,646 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:46,646 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:46,648 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:46,648 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:46,648 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:46,648 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/f43b997de8c34dd48bae12fd372f02b9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=47.9 K 2024-11-15T23:59:46,649 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f43b997de8c34dd48bae12fd372f02b9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:46,650 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:46,650 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 15302431f8aa412a82c2a7996d54189d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731715182781 2024-11-15T23:59:46,650 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:46,650 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:46,650 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/78c8a60a6c2e43c4a5a4aa854ca1c001, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=50.2 K 2024-11-15T23:59:46,650 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b6e4e7f5e841ad94d39d71659cb5e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731715183433 2024-11-15T23:59:46,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c8a60a6c2e43c4a5a4aa854ca1c001, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:46,658 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45d56a9ca2d546ea8666ca04f31aef77, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731715182781 2024-11-15T23:59:46,658 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 486d0e4a57fb4e3eafc4bea0c4076a49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:46,658 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1fb59c30ddd4de5bd929cafb8d0cc44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731715183433 2024-11-15T23:59:46,659 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0689cb10b01d4c2fa86df29088433c3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:46,672 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:46,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:46,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:59:46,673 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7f8ff3391e0a4dc4b8639699f267e4ea is 50, key is test_row_0/C:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:46,676 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:46,677 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/4c756051dc1447ae907dd1700caad576 is 50, key is test_row_0/A:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/7b070c72444c4def9d620d074c12d944 is 50, key is test_row_0/A:col10/1731715186666/Put/seqid=0 2024-11-15T23:59:46,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742107_1283 (size=12731) 2024-11-15T23:59:46,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742108_1284 (size=12731) 2024-11-15T23:59:46,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742109_1285 (size=14741) 2024-11-15T23:59:46,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/7b070c72444c4def9d620d074c12d944 2024-11-15T23:59:46,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e07055245817409489cff79539eba8a6 is 50, key is test_row_0/B:col10/1731715186666/Put/seqid=0 2024-11-15T23:59:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T23:59:46,733 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:46,734 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T23:59:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:46,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:46,737 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:46,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715246756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715246756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715246757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715246763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715246766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742110_1286 (size=12301) 2024-11-15T23:59:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715246869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715246869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715246880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715246880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715246880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:46,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:46,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715247080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715247081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715247089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715247089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715247090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,120 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7f8ff3391e0a4dc4b8639699f267e4ea as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7f8ff3391e0a4dc4b8639699f267e4ea 2024-11-15T23:59:47,121 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/4c756051dc1447ae907dd1700caad576 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/4c756051dc1447ae907dd1700caad576 2024-11-15T23:59:47,129 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 4c756051dc1447ae907dd1700caad576(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:47,129 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:47,129 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=12, startTime=1731715186646; duration=0sec 2024-11-15T23:59:47,130 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:47,130 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:47,130 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:47,132 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 7f8ff3391e0a4dc4b8639699f267e4ea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:47,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:47,132 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=12, startTime=1731715186646; duration=0sec 2024-11-15T23:59:47,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:47,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:47,133 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:47,133 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:47,133 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,133 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/937f0194c7844e93a815534cc2287971, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=47.9 K 2024-11-15T23:59:47,133 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937f0194c7844e93a815534cc2287971, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731715182454 2024-11-15T23:59:47,135 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69668ca57a648ba8e4d119256a74d79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731715182781 2024-11-15T23:59:47,135 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518e048e54244fb78575a291249fb470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731715183433 2024-11-15T23:59:47,136 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1236a651ab0f4dd0b2594c951a4616d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:47,169 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:47,171 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e517c79c8db74347b86ecbb2c5940b26 is 50, key is test_row_0/B:col10/1731715185992/Put/seqid=0 2024-11-15T23:59:47,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e07055245817409489cff79539eba8a6 2024-11-15T23:59:47,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742111_1287 (size=12731) 2024-11-15T23:59:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/3039119443624b1295e4d70780c03a92 is 50, key is test_row_0/C:col10/1731715186666/Put/seqid=0 2024-11-15T23:59:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742112_1288 (size=12301) 2024-11-15T23:59:47,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715247389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715247400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715247400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715247400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715247401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,613 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e517c79c8db74347b86ecbb2c5940b26 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e517c79c8db74347b86ecbb2c5940b26 2024-11-15T23:59:47,619 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into e517c79c8db74347b86ecbb2c5940b26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:47,619 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:47,619 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=12, startTime=1731715186646; duration=0sec 2024-11-15T23:59:47,620 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:47,620 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:47,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/3039119443624b1295e4d70780c03a92 2024-11-15T23:59:47,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/7b070c72444c4def9d620d074c12d944 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944 2024-11-15T23:59:47,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944, entries=200, sequenceid=264, filesize=14.4 K 2024-11-15T23:59:47,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e07055245817409489cff79539eba8a6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6 2024-11-15T23:59:47,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T23:59:47,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/3039119443624b1295e4d70780c03a92 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92 2024-11-15T23:59:47,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:47,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T23:59:47,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6582600b266a5542c0d5a9612f296646 in 1005ms, sequenceid=264, compaction requested=false 2024-11-15T23:59:47,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:47,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T23:59:47,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,823 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:47,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/f3742f5707fe470abc84856bb070a494 is 50, key is test_row_0/A:col10/1731715186761/Put/seqid=0 2024-11-15T23:59:47,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742113_1289 (size=12301) 2024-11-15T23:59:47,867 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/f3742f5707fe470abc84856bb070a494 2024-11-15T23:59:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4f07a39aad8b48c4a8c3b2c039f50aca is 50, key is test_row_0/B:col10/1731715186761/Put/seqid=0 2024-11-15T23:59:47,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742114_1290 (size=12301) 2024-11-15T23:59:47,901 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4f07a39aad8b48c4a8c3b2c039f50aca 2024-11-15T23:59:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:47,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:47,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/6410af765b3945cc9d337fefa2c8affb is 50, key is test_row_0/C:col10/1731715186761/Put/seqid=0 2024-11-15T23:59:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715247919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715247920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742115_1291 (size=12301) 2024-11-15T23:59:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715247923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715247923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715247924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:47,936 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/6410af765b3945cc9d337fefa2c8affb 2024-11-15T23:59:47,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/f3742f5707fe470abc84856bb070a494 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494 2024-11-15T23:59:47,948 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T23:59:47,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/4f07a39aad8b48c4a8c3b2c039f50aca as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca 2024-11-15T23:59:47,954 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T23:59:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/6410af765b3945cc9d337fefa2c8affb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb 2024-11-15T23:59:47,962 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T23:59:47,963 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 140ms, sequenceid=291, compaction requested=true 2024-11-15T23:59:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:47,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T23:59:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T23:59:47,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T23:59:47,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-11-15T23:59:47,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2340 sec 2024-11-15T23:59:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:48,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:59:48,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/2e608d8fe2444f2b9be8153090c44471 is 50, key is test_row_0/A:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:48,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742116_1292 (size=14741) 2024-11-15T23:59:48,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/2e608d8fe2444f2b9be8153090c44471 2024-11-15T23:59:48,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f885e07e849b4caaa1e14a60e715c40a is 50, key is test_row_0/B:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:48,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715248126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715248128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715248126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715248128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742117_1293 (size=12301) 2024-11-15T23:59:48,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f885e07e849b4caaa1e14a60e715c40a 2024-11-15T23:59:48,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/8901a77a24e848dbb129a79c2b64caa7 is 50, key is test_row_0/C:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:48,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742118_1294 (size=12301) 2024-11-15T23:59:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715248241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715248242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715248242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715248242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715248453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715248454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715248454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715248454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/8901a77a24e848dbb129a79c2b64caa7 2024-11-15T23:59:48,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/2e608d8fe2444f2b9be8153090c44471 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471 2024-11-15T23:59:48,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471, entries=200, sequenceid=304, filesize=14.4 K 2024-11-15T23:59:48,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/f885e07e849b4caaa1e14a60e715c40a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a 2024-11-15T23:59:48,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-15T23:59:48,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/8901a77a24e848dbb129a79c2b64caa7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7 2024-11-15T23:59:48,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7, entries=150, sequenceid=304, filesize=12.0 K 2024-11-15T23:59:48,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6582600b266a5542c0d5a9612f296646 in 611ms, sequenceid=304, compaction requested=true 2024-11-15T23:59:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:48,654 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:48,655 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:48,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:48,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:48,657 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:48,657 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/4c756051dc1447ae907dd1700caad576, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=53.2 K 2024-11-15T23:59:48,658 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c756051dc1447ae907dd1700caad576, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:48,658 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:48,658 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:48,658 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:48,658 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e517c79c8db74347b86ecbb2c5940b26, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=48.5 K 2024-11-15T23:59:48,658 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b070c72444c4def9d620d074c12d944, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731715186018 2024-11-15T23:59:48,659 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e517c79c8db74347b86ecbb2c5940b26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:48,660 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e07055245817409489cff79539eba8a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731715186019 2024-11-15T23:59:48,660 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3742f5707fe470abc84856bb070a494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731715186748 2024-11-15T23:59:48,661 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f07a39aad8b48c4a8c3b2c039f50aca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731715186748 2024-11-15T23:59:48,661 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e608d8fe2444f2b9be8153090c44471, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:48,661 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f885e07e849b4caaa1e14a60e715c40a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:48,676 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:48,677 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/7613da2c633248f19467d4d6520c5dc6 is 50, key is test_row_0/A:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:48,679 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:48,680 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad is 50, key is test_row_0/B:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:48,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742119_1295 (size=13017) 2024-11-15T23:59:48,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742120_1296 (size=13017) 2024-11-15T23:59:48,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:48,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T23:59:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:48,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715248792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9a250d2ff3674a7191d6efa6271a04d5 is 50, key is test_row_0/A:col10/1731715188776/Put/seqid=0 2024-11-15T23:59:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715248792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715248795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715248795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742121_1297 (size=12301) 2024-11-15T23:59:48,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9a250d2ff3674a7191d6efa6271a04d5 2024-11-15T23:59:48,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/a1f6a5e070304a9a9dd213ee3c152e82 is 50, key is test_row_0/B:col10/1731715188776/Put/seqid=0 2024-11-15T23:59:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T23:59:48,879 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:48,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T23:59:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:48,883 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:48,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:48,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:48,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742122_1298 (size=12301) 2024-11-15T23:59:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715248900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715248910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715248910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715248910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715248939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:48,944 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 6582600b266a5542c0d5a9612f296646 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:49,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715249110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715249120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715249121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715249121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,129 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad 2024-11-15T23:59:49,129 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/7613da2c633248f19467d4d6520c5dc6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7613da2c633248f19467d4d6520c5dc6 2024-11-15T23:59:49,142 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 7613da2c633248f19467d4d6520c5dc6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:49,142 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:49,142 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=12, startTime=1731715188654; duration=0sec 2024-11-15T23:59:49,143 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 70f71c0f2aa54fbeb98fb0c5a9ce85ad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:49,143 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:49,143 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=12, startTime=1731715188655; duration=0sec 2024-11-15T23:59:49,143 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:49,143 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:49,144 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T23:59:49,144 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:49,144 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:49,145 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T23:59:49,145 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:49,145 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,145 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7f8ff3391e0a4dc4b8639699f267e4ea, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=48.5 K 2024-11-15T23:59:49,146 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8ff3391e0a4dc4b8639699f267e4ea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731715185647 2024-11-15T23:59:49,146 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3039119443624b1295e4d70780c03a92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731715186019 2024-11-15T23:59:49,146 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6410af765b3945cc9d337fefa2c8affb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731715186748 2024-11-15T23:59:49,147 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8901a77a24e848dbb129a79c2b64caa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:49,164 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:49,165 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e83cf6ad301c4800a58472b539adde6b is 50, key is test_row_0/C:col10/1731715187921/Put/seqid=0 2024-11-15T23:59:49,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742123_1299 (size=13017) 2024-11-15T23:59:49,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:49,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/a1f6a5e070304a9a9dd213ee3c152e82 2024-11-15T23:59:49,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/ab4c7a24860d4fb897e7b86f441d0d94 is 50, key is test_row_0/C:col10/1731715188776/Put/seqid=0 2024-11-15T23:59:49,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742124_1300 (size=12301) 2024-11-15T23:59:49,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/ab4c7a24860d4fb897e7b86f441d0d94 2024-11-15T23:59:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/9a250d2ff3674a7191d6efa6271a04d5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5 2024-11-15T23:59:49,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T23:59:49,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/a1f6a5e070304a9a9dd213ee3c152e82 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82 2024-11-15T23:59:49,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T23:59:49,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/ab4c7a24860d4fb897e7b86f441d0d94 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94 2024-11-15T23:59:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T23:59:49,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 592ms, sequenceid=329, compaction requested=false 2024-11-15T23:59:49,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:49,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:49,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:59:49,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/ecb4b31ab26341e69359d7e912a86916 is 50, key is test_row_0/A:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:49,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742125_1301 (size=12301) 2024-11-15T23:59:49,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715249479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715249486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715249489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715249489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:49,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,577 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e83cf6ad301c4800a58472b539adde6b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e83cf6ad301c4800a58472b539adde6b 2024-11-15T23:59:49,590 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into e83cf6ad301c4800a58472b539adde6b(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:49,590 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:49,590 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=12, startTime=1731715188655; duration=0sec 2024-11-15T23:59:49,590 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:49,590 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:49,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715249600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715249601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715249601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715249601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:49,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715249813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715249813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715249813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715249813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:49,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/ecb4b31ab26341e69359d7e912a86916 2024-11-15T23:59:49,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1d5138fa50e343a5a7c812d9df27b43c is 50, key is test_row_0/B:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:49,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742126_1302 (size=12301) 2024-11-15T23:59:49,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1d5138fa50e343a5a7c812d9df27b43c 2024-11-15T23:59:49,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bd4f4336c8a34b35a91ea0111f74d1e7 is 50, key is test_row_0/C:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:49,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742127_1303 (size=12301) 2024-11-15T23:59:49,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bd4f4336c8a34b35a91ea0111f74d1e7 2024-11-15T23:59:49,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/ecb4b31ab26341e69359d7e912a86916 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916 2024-11-15T23:59:49,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T23:59:49,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/1d5138fa50e343a5a7c812d9df27b43c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c 2024-11-15T23:59:49,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T23:59:49,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bd4f4336c8a34b35a91ea0111f74d1e7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7 2024-11-15T23:59:49,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T23:59:49,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6582600b266a5542c0d5a9612f296646 in 549ms, sequenceid=344, compaction requested=true 2024-11-15T23:59:49,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:49,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:49,984 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:49,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:49,984 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:49,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:49,985 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:49,985 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:49,985 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,985 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7613da2c633248f19467d4d6520c5dc6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=36.7 K 2024-11-15T23:59:49,986 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:49,986 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:49,986 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,986 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=36.7 K 2024-11-15T23:59:49,986 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 70f71c0f2aa54fbeb98fb0c5a9ce85ad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:49,986 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7613da2c633248f19467d4d6520c5dc6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:49,987 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f6a5e070304a9a9dd213ee3c152e82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731715188122 2024-11-15T23:59:49,987 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a250d2ff3674a7191d6efa6271a04d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731715188122 2024-11-15T23:59:49,988 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5138fa50e343a5a7c812d9df27b43c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:49,988 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb4b31ab26341e69359d7e912a86916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:49,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T23:59:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:49,993 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d743e5133cda4fb48d7137ec6898e8e2 is 50, key is test_row_0/A:col10/1731715189483/Put/seqid=0 2024-11-15T23:59:50,011 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:50,012 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/30639ae4a22844fe9d7e4ce64571058c is 50, key is test_row_0/B:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:50,028 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:50,029 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/b6805d3518f84851b7dd06a5e17f6a38 is 50, key is test_row_0/A:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:50,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742128_1304 (size=12301) 2024-11-15T23:59:50,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742129_1305 (size=13119) 2024-11-15T23:59:50,053 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/30639ae4a22844fe9d7e4ce64571058c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/30639ae4a22844fe9d7e4ce64571058c 2024-11-15T23:59:50,064 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 30639ae4a22844fe9d7e4ce64571058c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:50,064 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:50,064 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715189984; duration=0sec 2024-11-15T23:59:50,064 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:50,064 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:50,064 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:50,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742130_1306 (size=13119) 2024-11-15T23:59:50,069 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:50,069 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:50,069 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:50,070 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e83cf6ad301c4800a58472b539adde6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=36.7 K 2024-11-15T23:59:50,070 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e83cf6ad301c4800a58472b539adde6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731715187918 2024-11-15T23:59:50,071 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4c7a24860d4fb897e7b86f441d0d94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731715188122 2024-11-15T23:59:50,075 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4f4336c8a34b35a91ea0111f74d1e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:50,076 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/b6805d3518f84851b7dd06a5e17f6a38 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b6805d3518f84851b7dd06a5e17f6a38 2024-11-15T23:59:50,084 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into b6805d3518f84851b7dd06a5e17f6a38(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:50,085 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:50,085 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715189984; duration=0sec 2024-11-15T23:59:50,085 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:50,085 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:50,088 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:50,088 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7472e5e7b86049828263509a288b122f is 50, key is test_row_0/C:col10/1731715188792/Put/seqid=0 2024-11-15T23:59:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:50,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:50,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742131_1307 (size=13119) 2024-11-15T23:59:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715250153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715250155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715250156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715250158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715250270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715250271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715250271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715250272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,442 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d743e5133cda4fb48d7137ec6898e8e2 2024-11-15T23:59:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5c3709cb04a54bbf8240f1a290703fd3 is 50, key is test_row_0/B:col10/1731715189483/Put/seqid=0 2024-11-15T23:59:50,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715250481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715250481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715250481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715250481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742132_1308 (size=12301) 2024-11-15T23:59:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5c3709cb04a54bbf8240f1a290703fd3 2024-11-15T23:59:50,541 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7472e5e7b86049828263509a288b122f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7472e5e7b86049828263509a288b122f 2024-11-15T23:59:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/512a3f6ac2554dbeae0032f506921ffc is 50, key is test_row_0/C:col10/1731715189483/Put/seqid=0 2024-11-15T23:59:50,548 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 7472e5e7b86049828263509a288b122f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:50,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:50,548 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715189985; duration=0sec 2024-11-15T23:59:50,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:50,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:50,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742133_1309 (size=12301) 2024-11-15T23:59:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715250790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715250790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715250791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715250791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715250961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:50,969 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 6582600b266a5542c0d5a9612f296646 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:50,978 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/512a3f6ac2554dbeae0032f506921ffc 2024-11-15T23:59:50,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/d743e5133cda4fb48d7137ec6898e8e2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2 2024-11-15T23:59:50,993 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2, entries=150, sequenceid=369, filesize=12.0 K 2024-11-15T23:59:50,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/5c3709cb04a54bbf8240f1a290703fd3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3 2024-11-15T23:59:50,999 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3, entries=150, sequenceid=369, filesize=12.0 K 2024-11-15T23:59:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/512a3f6ac2554dbeae0032f506921ffc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc 2024-11-15T23:59:51,021 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc, entries=150, sequenceid=369, filesize=12.0 K 2024-11-15T23:59:51,023 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6582600b266a5542c0d5a9612f296646 in 1029ms, sequenceid=369, compaction requested=false 2024-11-15T23:59:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T23:59:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T23:59:51,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T23:59:51,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1400 sec 2024-11-15T23:59:51,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.1480 sec 2024-11-15T23:59:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T23:59:51,030 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:51,031 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T23:59:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:59:51,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:51,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:51,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:59:51,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T23:59:51,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:51,189 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:59:51,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/571875561b094428bc69c9611c04798a is 50, key is test_row_0/A:col10/1731715190155/Put/seqid=0 2024-11-15T23:59:51,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742134_1310 (size=12301) 2024-11-15T23:59:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:51,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715251337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715251341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715251344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715251344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:59:51,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715251454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715251454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715251466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715251468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,640 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/571875561b094428bc69c9611c04798a 2024-11-15T23:59:51,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/2a6a2263309847f3a42f2f4e177c5ba7 is 50, key is test_row_0/B:col10/1731715190155/Put/seqid=0 2024-11-15T23:59:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:59:51,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715251660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715251660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715251673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742135_1311 (size=12301) 2024-11-15T23:59:51,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715251679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715251970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715251970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715251990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715251990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,079 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/2a6a2263309847f3a42f2f4e177c5ba7 2024-11-15T23:59:52,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/903f4f94c84a440bbad98cfca43ed213 is 50, key is test_row_0/C:col10/1731715190155/Put/seqid=0 2024-11-15T23:59:52,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742136_1312 (size=12301) 2024-11-15T23:59:52,129 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/903f4f94c84a440bbad98cfca43ed213 2024-11-15T23:59:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/571875561b094428bc69c9611c04798a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a 2024-11-15T23:59:52,142 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:59:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/2a6a2263309847f3a42f2f4e177c5ba7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7 2024-11-15T23:59:52,153 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:59:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/903f4f94c84a440bbad98cfca43ed213 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213 2024-11-15T23:59:52,160 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213, entries=150, sequenceid=384, filesize=12.0 K 2024-11-15T23:59:52,161 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6582600b266a5542c0d5a9612f296646 in 973ms, sequenceid=384, compaction requested=true 2024-11-15T23:59:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:52,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T23:59:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T23:59:52,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T23:59:52,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1290 sec 2024-11-15T23:59:52,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1360 sec 2024-11-15T23:59:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T23:59:52,170 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:52,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T23:59:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:59:52,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:52,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:52,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:59:52,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T23:59:52,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:52,328 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T23:59:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/85d0768b6b9442a59a6a617fa9bb85bf is 50, key is test_row_0/A:col10/1731715191342/Put/seqid=0 2024-11-15T23:59:52,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742137_1313 (size=12301) 2024-11-15T23:59:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:52,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:59:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715252509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715252544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715252544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715252544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715252659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715252660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715252660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715252660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,760 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/85d0768b6b9442a59a6a617fa9bb85bf 2024-11-15T23:59:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/243275e8e30e4c1998b0cb8b6946873f is 50, key is test_row_0/B:col10/1731715191342/Put/seqid=0 2024-11-15T23:59:52,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742138_1314 (size=12301) 2024-11-15T23:59:52,784 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/243275e8e30e4c1998b0cb8b6946873f 2024-11-15T23:59:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bf1c7bafc87e4cea959851f40ab89a76 is 50, key is test_row_0/C:col10/1731715191342/Put/seqid=0 2024-11-15T23:59:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:59:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742139_1315 (size=12301) 2024-11-15T23:59:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715252870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715252870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715252870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715252870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715253179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715253179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715253180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715253180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,204 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bf1c7bafc87e4cea959851f40ab89a76 2024-11-15T23:59:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/85d0768b6b9442a59a6a617fa9bb85bf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf 2024-11-15T23:59:53,220 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T23:59:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/243275e8e30e4c1998b0cb8b6946873f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f 2024-11-15T23:59:53,225 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T23:59:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/bf1c7bafc87e4cea959851f40ab89a76 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76 2024-11-15T23:59:53,231 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76, entries=150, sequenceid=409, filesize=12.0 K 2024-11-15T23:59:53,233 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 6582600b266a5542c0d5a9612f296646 in 904ms, sequenceid=409, compaction requested=true 2024-11-15T23:59:53,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:53,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:53,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T23:59:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T23:59:53,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T23:59:53,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0590 sec 2024-11-15T23:59:53,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0670 sec 2024-11-15T23:59:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T23:59:53,311 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:53,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T23:59:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:59:53,326 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:53,326 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:53,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:59:53,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T23:59:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:53,479 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T23:59:53,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/65100aa8401a48d6bdcd21cd81e6eb91 is 50, key is test_row_0/A:col10/1731715192498/Put/seqid=0 2024-11-15T23:59:53,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742140_1316 (size=12301) 2024-11-15T23:59:53,489 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/65100aa8401a48d6bdcd21cd81e6eb91 2024-11-15T23:59:53,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e3119daf1aa24c06a1200777b2faee6c is 50, key is test_row_0/B:col10/1731715192498/Put/seqid=0 2024-11-15T23:59:53,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742141_1317 (size=12301) 2024-11-15T23:59:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:59:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:53,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715253728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715253730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715253731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715253732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715253839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715253840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715253840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715253850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:53,911 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e3119daf1aa24c06a1200777b2faee6c 2024-11-15T23:59:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/80800a5d604442688e9e8c1161fa8396 is 50, key is test_row_0/C:col10/1731715192498/Put/seqid=0 2024-11-15T23:59:53,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742142_1318 (size=12301) 2024-11-15T23:59:53,941 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/80800a5d604442688e9e8c1161fa8396 2024-11-15T23:59:53,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/65100aa8401a48d6bdcd21cd81e6eb91 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91 2024-11-15T23:59:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:59:53,952 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91, entries=150, sequenceid=422, filesize=12.0 K 2024-11-15T23:59:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/e3119daf1aa24c06a1200777b2faee6c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c 2024-11-15T23:59:53,960 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c, entries=150, sequenceid=422, filesize=12.0 K 2024-11-15T23:59:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/80800a5d604442688e9e8c1161fa8396 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396 2024-11-15T23:59:53,965 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396, entries=150, sequenceid=422, filesize=12.0 K 2024-11-15T23:59:53,967 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6582600b266a5542c0d5a9612f296646 in 488ms, sequenceid=422, compaction requested=true 2024-11-15T23:59:53,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:53,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:53,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T23:59:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T23:59:53,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T23:59:53,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2024-11-15T23:59:53,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 659 msec 2024-11-15T23:59:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:54,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:54,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:54,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/156891a9280c4530862128320117ee73 is 50, key is test_row_0/A:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:54,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715254067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715254068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715254069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715254070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742143_1319 (size=14741) 2024-11-15T23:59:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715254179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715254390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715254393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715254393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715254393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T23:59:54,460 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:54,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T23:59:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:54,462 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:54,463 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:54,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:54,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/156891a9280c4530862128320117ee73 2024-11-15T23:59:54,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/6942bf1d4eb4409cbc6eb2e67dc81230 is 50, key is test_row_0/B:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:54,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742144_1320 (size=12301) 2024-11-15T23:59:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:54,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715254700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715254700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715254700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715254710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:54,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:54,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/6942bf1d4eb4409cbc6eb2e67dc81230 2024-11-15T23:59:54,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a13052b7ea70450a9afb703de7bb0504 is 50, key is test_row_0/C:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:54,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742145_1321 (size=12301) 2024-11-15T23:59:54,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:54,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49616 deadline: 1731715255000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:55,004 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 6582600b266a5542c0d5a9612f296646 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T23:59:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:55,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:55,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715255209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715255209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:55,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715255219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:55,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715255220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:55,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:55,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:55,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:55,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a13052b7ea70450a9afb703de7bb0504 2024-11-15T23:59:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/156891a9280c4530862128320117ee73 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73 2024-11-15T23:59:55,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73, entries=200, sequenceid=447, filesize=14.4 K 2024-11-15T23:59:55,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/6942bf1d4eb4409cbc6eb2e67dc81230 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230 2024-11-15T23:59:55,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230, entries=150, sequenceid=447, filesize=12.0 K 2024-11-15T23:59:55,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a13052b7ea70450a9afb703de7bb0504 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504 2024-11-15T23:59:55,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504, entries=150, sequenceid=447, filesize=12.0 K 2024-11-15T23:59:55,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6582600b266a5542c0d5a9612f296646 in 1282ms, sequenceid=447, compaction requested=true 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:55,335 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:55,335 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:59:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:55,337 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74624 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:59:55,337 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:55,337 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,337 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 77064 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:59:55,337 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:55,337 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/30639ae4a22844fe9d7e4ce64571058c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=72.9 K 2024-11-15T23:59:55,337 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,337 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b6805d3518f84851b7dd06a5e17f6a38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=75.3 K 2024-11-15T23:59:55,338 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6805d3518f84851b7dd06a5e17f6a38, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:55,338 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 30639ae4a22844fe9d7e4ce64571058c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:55,338 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3709cb04a54bbf8240f1a290703fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731715189466 2024-11-15T23:59:55,338 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d743e5133cda4fb48d7137ec6898e8e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731715189466 2024-11-15T23:59:55,339 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6a2263309847f3a42f2f4e177c5ba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715190153 2024-11-15T23:59:55,339 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571875561b094428bc69c9611c04798a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715190153 2024-11-15T23:59:55,339 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 243275e8e30e4c1998b0cb8b6946873f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731715191330 2024-11-15T23:59:55,339 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d0768b6b9442a59a6a617fa9bb85bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731715191330 2024-11-15T23:59:55,340 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65100aa8401a48d6bdcd21cd81e6eb91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731715192498 2024-11-15T23:59:55,340 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e3119daf1aa24c06a1200777b2faee6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731715192498 2024-11-15T23:59:55,340 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156891a9280c4530862128320117ee73, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:55,340 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 6942bf1d4eb4409cbc6eb2e67dc81230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:55,352 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:55,353 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/6513b873c5784e3cb702554b9bd0a064 is 50, key is test_row_0/A:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:55,359 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:55,360 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d555828378154fcf99e04668a1622312 is 50, key is test_row_0/B:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742146_1322 (size=13323) 2024-11-15T23:59:55,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742147_1323 (size=13323) 2024-11-15T23:59:55,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T23:59:55,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T23:59:55,385 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d555828378154fcf99e04668a1622312 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d555828378154fcf99e04668a1622312 2024-11-15T23:59:55,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a5c99e1540174ce58b6634b714509fc2 is 50, key is test_row_0/A:col10/1731715194067/Put/seqid=0 2024-11-15T23:59:55,392 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into d555828378154fcf99e04668a1622312(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:55,392 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:55,392 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=10, startTime=1731715195335; duration=0sec 2024-11-15T23:59:55,392 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:55,392 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:55,392 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T23:59:55,394 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74624 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T23:59:55,394 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:55,394 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,394 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7472e5e7b86049828263509a288b122f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=72.9 K 2024-11-15T23:59:55,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7472e5e7b86049828263509a288b122f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731715188790 2024-11-15T23:59:55,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 512a3f6ac2554dbeae0032f506921ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731715189466 2024-11-15T23:59:55,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 903f4f94c84a440bbad98cfca43ed213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715190153 2024-11-15T23:59:55,396 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1c7bafc87e4cea959851f40ab89a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731715191330 2024-11-15T23:59:55,396 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 80800a5d604442688e9e8c1161fa8396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731715192498 2024-11-15T23:59:55,396 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a13052b7ea70450a9afb703de7bb0504, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742148_1324 (size=12301) 2024-11-15T23:59:55,428 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a5c99e1540174ce58b6634b714509fc2 2024-11-15T23:59:55,456 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#276 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/715f0a12943d40d0a7284eeef6ef19f5 is 50, key is test_row_0/B:col10/1731715194067/Put/seqid=0 2024-11-15T23:59:55,457 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7c98be2ac6bd482d943731ea21508ddf is 50, key is test_row_0/C:col10/1731715194053/Put/seqid=0 2024-11-15T23:59:55,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742149_1325 (size=12301) 2024-11-15T23:59:55,497 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/715f0a12943d40d0a7284eeef6ef19f5 2024-11-15T23:59:55,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742150_1326 (size=13323) 2024-11-15T23:59:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/d8a3e8e53cb44a83b6b566d6f4a9af2b is 50, key is test_row_0/C:col10/1731715194067/Put/seqid=0 2024-11-15T23:59:55,553 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/7c98be2ac6bd482d943731ea21508ddf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7c98be2ac6bd482d943731ea21508ddf 2024-11-15T23:59:55,562 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 7c98be2ac6bd482d943731ea21508ddf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:55,562 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:55,562 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=10, startTime=1731715195335; duration=0sec 2024-11-15T23:59:55,562 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:55,562 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742151_1327 (size=12301) 2024-11-15T23:59:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:55,780 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/6513b873c5784e3cb702554b9bd0a064 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6513b873c5784e3cb702554b9bd0a064 2024-11-15T23:59:55,785 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into 6513b873c5784e3cb702554b9bd0a064(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:55,785 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:55,785 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=10, startTime=1731715195335; duration=0sec 2024-11-15T23:59:55,786 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:55,786 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:55,967 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/d8a3e8e53cb44a83b6b566d6f4a9af2b 2024-11-15T23:59:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/a5c99e1540174ce58b6634b714509fc2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2 2024-11-15T23:59:55,978 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2, entries=150, sequenceid=458, filesize=12.0 K 2024-11-15T23:59:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/715f0a12943d40d0a7284eeef6ef19f5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5 2024-11-15T23:59:55,984 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5, entries=150, sequenceid=458, filesize=12.0 K 2024-11-15T23:59:55,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/d8a3e8e53cb44a83b6b566d6f4a9af2b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b 2024-11-15T23:59:55,993 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b, entries=150, sequenceid=458, filesize=12.0 K 2024-11-15T23:59:55,994 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 6582600b266a5542c0d5a9612f296646 in 612ms, sequenceid=458, compaction requested=false 2024-11-15T23:59:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:55,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T23:59:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T23:59:55,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T23:59:55,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-15T23:59:56,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5400 sec 2024-11-15T23:59:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:56,252 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:56,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T23:59:56,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/42a99a37fb8f43a2a5f1c297b8c77f33 is 50, key is test_row_0/A:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:56,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742152_1328 (size=12301) 2024-11-15T23:59:56,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/42a99a37fb8f43a2a5f1c297b8c77f33 2024-11-15T23:59:56,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/04fc5ff8e510401882fc3c893cb1eab8 is 50, key is test_row_0/B:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:56,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715256317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715256320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715256322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715256323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742153_1329 (size=12301) 2024-11-15T23:59:56,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715256430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715256430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715256439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715256440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T23:59:56,610 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:56,611 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T23:59:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T23:59:56,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T23:59:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:56,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T23:59:56,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T23:59:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715256640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715256645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715256649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715256650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:56,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/04fc5ff8e510401882fc3c893cb1eab8 2024-11-15T23:59:56,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/c40c400c272e4eb5b19d79920e53e843 is 50, key is test_row_0/C:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:56,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:59:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:56,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742154_1330 (size=12301) 2024-11-15T23:59:56,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:59:56,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:56,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:56,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:56,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715256949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715256960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715256960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715256960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:59:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T23:59:57,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/c40c400c272e4eb5b19d79920e53e843 2024-11-15T23:59:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/42a99a37fb8f43a2a5f1c297b8c77f33 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33 2024-11-15T23:59:57,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33, entries=150, sequenceid=473, filesize=12.0 K 2024-11-15T23:59:57,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/04fc5ff8e510401882fc3c893cb1eab8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8 2024-11-15T23:59:57,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8, entries=150, sequenceid=473, filesize=12.0 K 2024-11-15T23:59:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/c40c400c272e4eb5b19d79920e53e843 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843 2024-11-15T23:59:57,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843, entries=150, sequenceid=473, filesize=12.0 K 2024-11-15T23:59:57,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6582600b266a5542c0d5a9612f296646 in 954ms, sequenceid=473, compaction requested=true 2024-11-15T23:59:57,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:57,208 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:57,208 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:57,209 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:57,209 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:57,209 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,209 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6513b873c5784e3cb702554b9bd0a064, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.0 K 2024-11-15T23:59:57,209 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:57,209 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:57,210 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,210 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d555828378154fcf99e04668a1622312, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.0 K 2024-11-15T23:59:57,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6513b873c5784e3cb702554b9bd0a064, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:57,210 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d555828378154fcf99e04668a1622312, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:57,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c99e1540174ce58b6634b714509fc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731715194061 2024-11-15T23:59:57,210 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 715f0a12943d40d0a7284eeef6ef19f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731715194061 2024-11-15T23:59:57,211 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42a99a37fb8f43a2a5f1c297b8c77f33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-15T23:59:57,211 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fc5ff8e510401882fc3c893cb1eab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-15T23:59:57,220 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:57,220 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/c3a750843484437ba11153558853b0a9 is 50, key is test_row_0/A:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:57,226 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:57,227 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/b8ffe68afb2e49aaa75f50e5e6c0f773 is 50, key is test_row_0/B:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:57,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T23:59:57,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,231 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T23:59:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:57,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742155_1331 (size=13425) 2024-11-15T23:59:57,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/bd5ae4d28d1646b79716c27aa8a01e5a is 50, key is test_row_0/A:col10/1731715196320/Put/seqid=0 2024-11-15T23:59:57,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742156_1332 (size=13425) 2024-11-15T23:59:57,275 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/c3a750843484437ba11153558853b0a9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/c3a750843484437ba11153558853b0a9 2024-11-15T23:59:57,301 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/b8ffe68afb2e49aaa75f50e5e6c0f773 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/b8ffe68afb2e49aaa75f50e5e6c0f773 2024-11-15T23:59:57,301 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into c3a750843484437ba11153558853b0a9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:57,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:57,301 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715197207; duration=0sec 2024-11-15T23:59:57,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:57,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-15T23:59:57,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:57,302 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:57,302 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-15T23:59:57,302 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:57,303 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7c98be2ac6bd482d943731ea21508ddf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.0 K 2024-11-15T23:59:57,303 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c98be2ac6bd482d943731ea21508ddf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731715193720 2024-11-15T23:59:57,303 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a3e8e53cb44a83b6b566d6f4a9af2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731715194061 2024-11-15T23:59:57,304 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c40c400c272e4eb5b19d79920e53e843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-15T23:59:57,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742157_1333 (size=12301) 2024-11-15T23:59:57,310 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into b8ffe68afb2e49aaa75f50e5e6c0f773(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:57,310 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:57,310 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715197208; duration=0sec 2024-11-15T23:59:57,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:57,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-15T23:59:57,314 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:57,314 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/ce10655e2f684b4087ed9d1918b6e238 is 50, key is test_row_0/C:col10/1731715196249/Put/seqid=0 2024-11-15T23:59:57,317 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/bd5ae4d28d1646b79716c27aa8a01e5a 2024-11-15T23:59:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c34c6a4e53b843ea9dd95070d79c23ac is 50, key is test_row_0/B:col10/1731715196320/Put/seqid=0 2024-11-15T23:59:57,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742158_1334 (size=13425) 2024-11-15T23:59:57,365 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/ce10655e2f684b4087ed9d1918b6e238 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ce10655e2f684b4087ed9d1918b6e238 2024-11-15T23:59:57,371 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into ce10655e2f684b4087ed9d1918b6e238(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T23:59:57,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:57,371 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715197208; duration=0sec 2024-11-15T23:59:57,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:57,371 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-15T23:59:57,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742159_1335 (size=12301) 2024-11-15T23:59:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:57,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. as already flushing 2024-11-15T23:59:57,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715257477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715257482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715257483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715257483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715257590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715257600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715257601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715257601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:57,778 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c34c6a4e53b843ea9dd95070d79c23ac 2024-11-15T23:59:57,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e3e9cdb32e48423c93efa97f11221ece is 50, key is test_row_0/C:col10/1731715196320/Put/seqid=0 2024-11-15T23:59:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742160_1336 (size=12301) 2024-11-15T23:59:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715257800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715257810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715257813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715257813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:57,966 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T23:59:58,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49600 deadline: 1731715258110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49644 deadline: 1731715258119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49592 deadline: 1731715258119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49650 deadline: 1731715258119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T23:59:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-15T23:59:58,200 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e3e9cdb32e48423c93efa97f11221ece 2024-11-15T23:59:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/bd5ae4d28d1646b79716c27aa8a01e5a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a 2024-11-15T23:59:58,215 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T23:59:58,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/c34c6a4e53b843ea9dd95070d79c23ac as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac 2024-11-15T23:59:58,226 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T23:59:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/e3e9cdb32e48423c93efa97f11221ece as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece 2024-11-15T23:59:58,233 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T23:59:58,234 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 6582600b266a5542c0d5a9612f296646 in 1002ms, sequenceid=497, compaction requested=false 2024-11-15T23:59:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T23:59:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T23:59:58,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T23:59:58,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-15T23:59:58,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6270 sec 2024-11-15T23:59:58,248 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-15T23:59:58,248 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,248 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,249 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-15T23:59:58,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,249 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,249 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,249 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,249 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,251 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-11-15T23:59:58,251 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,251 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,251 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-11-15T23:59:58,251 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,251 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-15T23:59:58,251 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,251 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,251 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,251 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,252 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,252 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 6582600b266a5542c0d5a9612f296646 2024-11-15T23:59:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T23:59:58,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T23:59:58,621 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-15T23:59:58,621 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,621 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,621 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/017ed348ed714f4eb2c444a56d336c48 is 50, key is test_row_0/A:col10/1731715197468/Put/seqid=0 2024-11-15T23:59:58,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742161_1337 (size=12301) 2024-11-15T23:59:58,632 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-15T23:59:58,632 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-15T23:59:58,632 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,632 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,632 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,632 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,632 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,632 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,633 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-15T23:59:58,633 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T23:59:58,633 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,633 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:59:58,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T23:59:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T23:59:58,760 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T23:59:59,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/017ed348ed714f4eb2c444a56d336c48 2024-11-15T23:59:59,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/88c7ccff1e944ee8b848c2231e90f860 is 50, key is test_row_0/B:col10/1731715197468/Put/seqid=0 2024-11-15T23:59:59,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742162_1338 (size=12301) 2024-11-15T23:59:59,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/88c7ccff1e944ee8b848c2231e90f860 2024-11-15T23:59:59,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/33ed31d8747b497c9349c79d0550de3c is 50, key is test_row_0/C:col10/1731715197468/Put/seqid=0 2024-11-15T23:59:59,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742163_1339 (size=12301) 2024-11-15T23:59:59,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/33ed31d8747b497c9349c79d0550de3c 2024-11-15T23:59:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/017ed348ed714f4eb2c444a56d336c48 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48 2024-11-15T23:59:59,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T23:59:59,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/88c7ccff1e944ee8b848c2231e90f860 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860 2024-11-15T23:59:59,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T23:59:59,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/33ed31d8747b497c9349c79d0550de3c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c 2024-11-15T23:59:59,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T23:59:59,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for 6582600b266a5542c0d5a9612f296646 in 1254ms, sequenceid=514, compaction requested=true 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6582600b266a5542c0d5a9612f296646:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T23:59:59,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T23:59:59,875 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:59,875 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T23:59:59,876 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:59,876 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T23:59:59,876 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/B is initiating minor compaction (all files) 2024-11-15T23:59:59,876 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/A is initiating minor compaction (all files) 2024-11-15T23:59:59,876 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/B in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:59,876 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/A in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-15T23:59:59,876 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/b8ffe68afb2e49aaa75f50e5e6c0f773, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.1 K 2024-11-15T23:59:59,877 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/c3a750843484437ba11153558853b0a9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.1 K 2024-11-15T23:59:59,877 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ffe68afb2e49aaa75f50e5e6c0f773, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-15T23:59:59,877 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a750843484437ba11153558853b0a9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-15T23:59:59,877 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c34c6a4e53b843ea9dd95070d79c23ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731715196311 2024-11-15T23:59:59,877 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5ae4d28d1646b79716c27aa8a01e5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731715196311 2024-11-15T23:59:59,877 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c7ccff1e944ee8b848c2231e90f860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731715197468 2024-11-15T23:59:59,878 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 017ed348ed714f4eb2c444a56d336c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731715197468 2024-11-15T23:59:59,884 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:59,884 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/e5de718bc0f74d11b8cfa18392f4380b is 50, key is test_row_0/A:col10/1731715197468/Put/seqid=0 2024-11-15T23:59:59,887 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T23:59:59,887 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/7f87029b593c428586f9f342a48aab45 is 50, key is test_row_0/B:col10/1731715197468/Put/seqid=0 2024-11-15T23:59:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742164_1340 (size=13527) 2024-11-15T23:59:59,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742165_1341 (size=13527) 2024-11-16T00:00:00,295 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/e5de718bc0f74d11b8cfa18392f4380b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/e5de718bc0f74d11b8cfa18392f4380b 2024-11-16T00:00:00,301 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/A of 6582600b266a5542c0d5a9612f296646 into e5de718bc0f74d11b8cfa18392f4380b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:00,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-16T00:00:00,301 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/A, priority=13, startTime=1731715199875; duration=0sec 2024-11-16T00:00:00,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:00,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:A 2024-11-16T00:00:00,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:00,302 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:00,302 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 6582600b266a5542c0d5a9612f296646/C is initiating minor compaction (all files) 2024-11-16T00:00:00,302 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6582600b266a5542c0d5a9612f296646/C in TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-16T00:00:00,302 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ce10655e2f684b4087ed9d1918b6e238, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp, totalSize=37.1 K 2024-11-16T00:00:00,303 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce10655e2f684b4087ed9d1918b6e238, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731715196246 2024-11-16T00:00:00,303 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e9cdb32e48423c93efa97f11221ece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731715196311 2024-11-16T00:00:00,303 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ed31d8747b497c9349c79d0550de3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731715197468 2024-11-16T00:00:00,305 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/7f87029b593c428586f9f342a48aab45 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/7f87029b593c428586f9f342a48aab45 2024-11-16T00:00:00,310 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6582600b266a5542c0d5a9612f296646#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:00,310 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/B of 6582600b266a5542c0d5a9612f296646 into 7f87029b593c428586f9f342a48aab45(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:00,310 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-16T00:00:00,310 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/B, priority=13, startTime=1731715199875; duration=0sec 2024-11-16T00:00:00,310 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:00,310 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:B 2024-11-16T00:00:00,311 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/891bd5f4ab3345519dbec64b23250606 is 50, key is test_row_0/C:col10/1731715197468/Put/seqid=0 2024-11-16T00:00:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742166_1342 (size=13527) 2024-11-16T00:00:00,321 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/891bd5f4ab3345519dbec64b23250606 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/891bd5f4ab3345519dbec64b23250606 2024-11-16T00:00:00,326 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6582600b266a5542c0d5a9612f296646/C of 6582600b266a5542c0d5a9612f296646 into 891bd5f4ab3345519dbec64b23250606(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:00,326 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6582600b266a5542c0d5a9612f296646: 2024-11-16T00:00:00,326 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., storeName=6582600b266a5542c0d5a9612f296646/C, priority=13, startTime=1731715199875; duration=0sec 2024-11-16T00:00:00,326 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:00,326 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6582600b266a5542c0d5a9612f296646:C 2024-11-16T00:00:05,021 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-16T00:00:05,021 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:05,021 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:05,021 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:05,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2543 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2589 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2610 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2527 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2568 2024-11-16T00:00:05,022 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T00:00:05,022 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:00:05,022 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:00:05,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:05,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:05,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:05,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-16T00:00:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:05,027 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715205027"}]},"ts":"1731715205027"} 2024-11-16T00:00:05,028 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T00:00:05,028 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T00:00:05,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:00:05,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, UNASSIGN}] 2024-11-16T00:00:05,031 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, UNASSIGN 2024-11-16T00:00:05,032 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=6582600b266a5542c0d5a9612f296646, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:05,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, UNASSIGN because future has completed 2024-11-16T00:00:05,034 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:00:05,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:05,186 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 6582600b266a5542c0d5a9612f296646 2024-11-16T00:00:05,186 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:00:05,187 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 6582600b266a5542c0d5a9612f296646, disabling compactions & flushes 2024-11-16T00:00:05,187 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-16T00:00:05,187 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-16T00:00:05,187 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. after waiting 0 ms 2024-11-16T00:00:05,187 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-16T00:00:05,187 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 6582600b266a5542c0d5a9612f296646 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T00:00:05,191 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/1cfb5572ecf043dfbab98e99874031f3 is 50, key is test_row_0/A:col10/1731715205020/Put/seqid=0 2024-11-16T00:00:05,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742167_1343 (size=9857) 2024-11-16T00:00:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:05,598 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/1cfb5572ecf043dfbab98e99874031f3 2024-11-16T00:00:05,605 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d09d7d591f384554acc50cff229b2c88 is 50, key is test_row_0/B:col10/1731715205020/Put/seqid=0 2024-11-16T00:00:05,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742168_1344 (size=9857) 2024-11-16T00:00:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:06,010 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d09d7d591f384554acc50cff229b2c88 2024-11-16T00:00:06,018 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a9fe993087db4bd788a9b03f0b371aeb is 50, key is test_row_0/C:col10/1731715205020/Put/seqid=0 2024-11-16T00:00:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742169_1345 (size=9857) 2024-11-16T00:00:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:06,422 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a9fe993087db4bd788a9b03f0b371aeb 2024-11-16T00:00:06,428 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/A/1cfb5572ecf043dfbab98e99874031f3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/1cfb5572ecf043dfbab98e99874031f3 2024-11-16T00:00:06,432 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/1cfb5572ecf043dfbab98e99874031f3, entries=100, sequenceid=524, filesize=9.6 K 2024-11-16T00:00:06,432 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/B/d09d7d591f384554acc50cff229b2c88 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d09d7d591f384554acc50cff229b2c88 2024-11-16T00:00:06,436 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d09d7d591f384554acc50cff229b2c88, entries=100, sequenceid=524, filesize=9.6 K 2024-11-16T00:00:06,436 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/.tmp/C/a9fe993087db4bd788a9b03f0b371aeb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a9fe993087db4bd788a9b03f0b371aeb 2024-11-16T00:00:06,440 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a9fe993087db4bd788a9b03f0b371aeb, entries=100, sequenceid=524, filesize=9.6 K 2024-11-16T00:00:06,441 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6582600b266a5542c0d5a9612f296646 in 1253ms, sequenceid=524, compaction requested=false 2024-11-16T00:00:06,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/5b5413efd9c64502a66d003efb3f634d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6ad52222f3444ed29c3e539b6637b55c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a6cea55cd66a4264807b72bd949b0421, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/78c8a60a6c2e43c4a5a4aa854ca1c001, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/4c756051dc1447ae907dd1700caad576, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7613da2c633248f19467d4d6520c5dc6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b6805d3518f84851b7dd06a5e17f6a38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6513b873c5784e3cb702554b9bd0a064, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/c3a750843484437ba11153558853b0a9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48] to archive 2024-11-16T00:00:06,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:06,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/3f55a073197c4bcfa132a40688893811 2024-11-16T00:00:06,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9fe53a50e8c7449d96b0a5430de01235 2024-11-16T00:00:06,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/27878fc9914c4e3aaa2f4506bcdc6032 2024-11-16T00:00:06,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/5b5413efd9c64502a66d003efb3f634d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/5b5413efd9c64502a66d003efb3f634d 2024-11-16T00:00:06,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/485a21f77cf447b9872056b0f1f58a17 2024-11-16T00:00:06,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d5140636be314e07a3a6826766df297d 2024-11-16T00:00:06,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a70a1a1ab6424f06bceba4205ea33349 2024-11-16T00:00:06,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6ad52222f3444ed29c3e539b6637b55c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6ad52222f3444ed29c3e539b6637b55c 2024-11-16T00:00:06,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b32619ed57524530bf6e85d4544c921b 2024-11-16T00:00:06,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/41e1a819f447440995f97c8f6b2b0a0d 2024-11-16T00:00:06,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a6cea55cd66a4264807b72bd949b0421 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a6cea55cd66a4264807b72bd949b0421 2024-11-16T00:00:06,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d762c903b938403b8a20022bbf7b39e3 2024-11-16T00:00:06,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/99a71dee221c4f51bfb1d7eec2f52cf3 2024-11-16T00:00:06,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/78c8a60a6c2e43c4a5a4aa854ca1c001 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/78c8a60a6c2e43c4a5a4aa854ca1c001 2024-11-16T00:00:06,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/45d56a9ca2d546ea8666ca04f31aef77 2024-11-16T00:00:06,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d1fb59c30ddd4de5bd929cafb8d0cc44 2024-11-16T00:00:06,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/4c756051dc1447ae907dd1700caad576 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/4c756051dc1447ae907dd1700caad576 2024-11-16T00:00:06,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/0689cb10b01d4c2fa86df29088433c3f 2024-11-16T00:00:06,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7b070c72444c4def9d620d074c12d944 2024-11-16T00:00:06,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/f3742f5707fe470abc84856bb070a494 2024-11-16T00:00:06,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/2e608d8fe2444f2b9be8153090c44471 2024-11-16T00:00:06,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7613da2c633248f19467d4d6520c5dc6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/7613da2c633248f19467d4d6520c5dc6 2024-11-16T00:00:06,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/9a250d2ff3674a7191d6efa6271a04d5 2024-11-16T00:00:06,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b6805d3518f84851b7dd06a5e17f6a38 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/b6805d3518f84851b7dd06a5e17f6a38 2024-11-16T00:00:06,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/ecb4b31ab26341e69359d7e912a86916 2024-11-16T00:00:06,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/d743e5133cda4fb48d7137ec6898e8e2 2024-11-16T00:00:06,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/571875561b094428bc69c9611c04798a 2024-11-16T00:00:06,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/85d0768b6b9442a59a6a617fa9bb85bf 2024-11-16T00:00:06,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/65100aa8401a48d6bdcd21cd81e6eb91 2024-11-16T00:00:06,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/156891a9280c4530862128320117ee73 2024-11-16T00:00:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6513b873c5784e3cb702554b9bd0a064 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/6513b873c5784e3cb702554b9bd0a064 2024-11-16T00:00:06,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/a5c99e1540174ce58b6634b714509fc2 2024-11-16T00:00:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/c3a750843484437ba11153558853b0a9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/c3a750843484437ba11153558853b0a9 2024-11-16T00:00:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/42a99a37fb8f43a2a5f1c297b8c77f33 2024-11-16T00:00:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/bd5ae4d28d1646b79716c27aa8a01e5a 2024-11-16T00:00:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/017ed348ed714f4eb2c444a56d336c48 2024-11-16T00:00:06,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9497992df53e482ca042864cf1a9f28d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/76d8e68203a94ace84c4c87d4d465806, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/da4ceb7e94fc42989286a8f075b8b135, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/937f0194c7844e93a815534cc2287971, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e517c79c8db74347b86ecbb2c5940b26, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/30639ae4a22844fe9d7e4ce64571058c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d555828378154fcf99e04668a1622312, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/b8ffe68afb2e49aaa75f50e5e6c0f773, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860] to archive 2024-11-16T00:00:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c085481af8764ca7ba7608d226433d0b 2024-11-16T00:00:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d6895795a298409e845bcf4cdd08ac38 2024-11-16T00:00:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5b7d35117ddb435f9d1781cb6b4fe663 2024-11-16T00:00:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9497992df53e482ca042864cf1a9f28d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9497992df53e482ca042864cf1a9f28d 2024-11-16T00:00:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4750ee4c84e848cca5ecf3372eba7f4a 2024-11-16T00:00:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d2ab5deab18c422997d51350ba4c20c1 2024-11-16T00:00:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/76d8e68203a94ace84c4c87d4d465806 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/76d8e68203a94ace84c4c87d4d465806 2024-11-16T00:00:06,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c28be5275f0f4f0f8ba75ef599ffe0e9 2024-11-16T00:00:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/454d06edf9c5437eb600962c416308e2 2024-11-16T00:00:06,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/da4ceb7e94fc42989286a8f075b8b135 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/da4ceb7e94fc42989286a8f075b8b135 2024-11-16T00:00:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/58cb5f3af44e4806b2b557d6630dc917 2024-11-16T00:00:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f3edb806c64c4477870aad361b908eb0 2024-11-16T00:00:06,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/937f0194c7844e93a815534cc2287971 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/937f0194c7844e93a815534cc2287971 2024-11-16T00:00:06,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/9d2b01199f344cabb406302864667e3a 2024-11-16T00:00:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f69668ca57a648ba8e4d119256a74d79 2024-11-16T00:00:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/518e048e54244fb78575a291249fb470 2024-11-16T00:00:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e517c79c8db74347b86ecbb2c5940b26 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e517c79c8db74347b86ecbb2c5940b26 2024-11-16T00:00:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1236a651ab0f4dd0b2594c951a4616d3 2024-11-16T00:00:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e07055245817409489cff79539eba8a6 2024-11-16T00:00:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/4f07a39aad8b48c4a8c3b2c039f50aca 2024-11-16T00:00:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/70f71c0f2aa54fbeb98fb0c5a9ce85ad 2024-11-16T00:00:06,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/f885e07e849b4caaa1e14a60e715c40a 2024-11-16T00:00:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/a1f6a5e070304a9a9dd213ee3c152e82 2024-11-16T00:00:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/30639ae4a22844fe9d7e4ce64571058c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/30639ae4a22844fe9d7e4ce64571058c 2024-11-16T00:00:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/1d5138fa50e343a5a7c812d9df27b43c 2024-11-16T00:00:06,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/5c3709cb04a54bbf8240f1a290703fd3 2024-11-16T00:00:06,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/2a6a2263309847f3a42f2f4e177c5ba7 2024-11-16T00:00:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/243275e8e30e4c1998b0cb8b6946873f 2024-11-16T00:00:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/e3119daf1aa24c06a1200777b2faee6c 2024-11-16T00:00:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d555828378154fcf99e04668a1622312 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d555828378154fcf99e04668a1622312 2024-11-16T00:00:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/6942bf1d4eb4409cbc6eb2e67dc81230 2024-11-16T00:00:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/715f0a12943d40d0a7284eeef6ef19f5 2024-11-16T00:00:06,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/b8ffe68afb2e49aaa75f50e5e6c0f773 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/b8ffe68afb2e49aaa75f50e5e6c0f773 2024-11-16T00:00:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/04fc5ff8e510401882fc3c893cb1eab8 2024-11-16T00:00:06,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/c34c6a4e53b843ea9dd95070d79c23ac 2024-11-16T00:00:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/88c7ccff1e944ee8b848c2231e90f860 2024-11-16T00:00:06,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/dbe2940ce4b7416f800694246a60a678, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/20fb765d80f34c33b24ba9be9c47e704, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/4c2888f8b0b54d708ab5770a538c96bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/f43b997de8c34dd48bae12fd372f02b9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7f8ff3391e0a4dc4b8639699f267e4ea, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e83cf6ad301c4800a58472b539adde6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7472e5e7b86049828263509a288b122f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7c98be2ac6bd482d943731ea21508ddf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ce10655e2f684b4087ed9d1918b6e238, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c] to archive 2024-11-16T00:00:06,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e840511b125a4b429c54a56e9807b186 2024-11-16T00:00:06,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/b50987d7691d4450b6930d607646bf2f 2024-11-16T00:00:06,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/57b50759e3194a16944b4e23bf04d0c4 2024-11-16T00:00:06,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/dbe2940ce4b7416f800694246a60a678 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/dbe2940ce4b7416f800694246a60a678 2024-11-16T00:00:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a6ddae5aa3534744b58147b3e76be7ec 2024-11-16T00:00:06,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/55fc2a92686f4c16a316493a304bd8d6 2024-11-16T00:00:06,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/20fb765d80f34c33b24ba9be9c47e704 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/20fb765d80f34c33b24ba9be9c47e704 2024-11-16T00:00:06,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/073cc293d5e74d679b0f65880f0fc32b 2024-11-16T00:00:06,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/936d0641037d42ad9da33dd4ca6307e5 2024-11-16T00:00:06,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/4c2888f8b0b54d708ab5770a538c96bf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/4c2888f8b0b54d708ab5770a538c96bf 2024-11-16T00:00:06,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/829f167a024b479294191a85b3adfe00 2024-11-16T00:00:06,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/43f842b751cb44d7af5b0284b60e6e57 2024-11-16T00:00:06,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/f43b997de8c34dd48bae12fd372f02b9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/f43b997de8c34dd48bae12fd372f02b9 2024-11-16T00:00:06,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/311d8e2328ff4953b2a85d124f809d68 2024-11-16T00:00:06,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/15302431f8aa412a82c2a7996d54189d 2024-11-16T00:00:06,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/18b6e4e7f5e841ad94d39d71659cb5e3 2024-11-16T00:00:06,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7f8ff3391e0a4dc4b8639699f267e4ea to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7f8ff3391e0a4dc4b8639699f267e4ea 2024-11-16T00:00:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/486d0e4a57fb4e3eafc4bea0c4076a49 2024-11-16T00:00:06,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/3039119443624b1295e4d70780c03a92 2024-11-16T00:00:06,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/6410af765b3945cc9d337fefa2c8affb 2024-11-16T00:00:06,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e83cf6ad301c4800a58472b539adde6b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e83cf6ad301c4800a58472b539adde6b 2024-11-16T00:00:06,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/8901a77a24e848dbb129a79c2b64caa7 2024-11-16T00:00:06,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ab4c7a24860d4fb897e7b86f441d0d94 2024-11-16T00:00:06,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7472e5e7b86049828263509a288b122f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7472e5e7b86049828263509a288b122f 2024-11-16T00:00:06,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bd4f4336c8a34b35a91ea0111f74d1e7 2024-11-16T00:00:06,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/512a3f6ac2554dbeae0032f506921ffc 2024-11-16T00:00:06,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/903f4f94c84a440bbad98cfca43ed213 2024-11-16T00:00:06,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/bf1c7bafc87e4cea959851f40ab89a76 2024-11-16T00:00:06,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/80800a5d604442688e9e8c1161fa8396 2024-11-16T00:00:06,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7c98be2ac6bd482d943731ea21508ddf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/7c98be2ac6bd482d943731ea21508ddf 2024-11-16T00:00:06,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a13052b7ea70450a9afb703de7bb0504 2024-11-16T00:00:06,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/d8a3e8e53cb44a83b6b566d6f4a9af2b 2024-11-16T00:00:06,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ce10655e2f684b4087ed9d1918b6e238 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/ce10655e2f684b4087ed9d1918b6e238 2024-11-16T00:00:06,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/c40c400c272e4eb5b19d79920e53e843 2024-11-16T00:00:06,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/e3e9cdb32e48423c93efa97f11221ece 2024-11-16T00:00:06,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/33ed31d8747b497c9349c79d0550de3c 2024-11-16T00:00:06,569 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/recovered.edits/527.seqid, newMaxSeqId=527, maxSeqId=1 2024-11-16T00:00:06,570 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. 2024-11-16T00:00:06,570 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 6582600b266a5542c0d5a9612f296646: Waiting for close lock at 1731715205186Running coprocessor pre-close hooks at 1731715205186Disabling compacts and flushes for region at 1731715205186Disabling writes for close at 1731715205187 (+1 ms)Obtaining lock to block concurrent updates at 1731715205187Preparing flush snapshotting stores in 6582600b266a5542c0d5a9612f296646 at 1731715205187Finished memstore snapshotting TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731715205187Flushing stores of TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646. at 1731715205188 (+1 ms)Flushing 6582600b266a5542c0d5a9612f296646/A: creating writer at 1731715205188Flushing 6582600b266a5542c0d5a9612f296646/A: appending metadata at 1731715205191 (+3 ms)Flushing 6582600b266a5542c0d5a9612f296646/A: closing flushed file at 1731715205191Flushing 6582600b266a5542c0d5a9612f296646/B: creating writer at 1731715205602 (+411 ms)Flushing 6582600b266a5542c0d5a9612f296646/B: appending metadata at 1731715205605 (+3 ms)Flushing 6582600b266a5542c0d5a9612f296646/B: closing flushed file at 1731715205605Flushing 6582600b266a5542c0d5a9612f296646/C: creating writer at 1731715206014 (+409 ms)Flushing 6582600b266a5542c0d5a9612f296646/C: appending metadata at 1731715206017 (+3 ms)Flushing 6582600b266a5542c0d5a9612f296646/C: closing flushed file at 1731715206017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bfd2c52: reopening flushed file at 1731715206427 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2378ba1: reopening flushed file at 1731715206432 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2337e87c: reopening flushed file at 1731715206436 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6582600b266a5542c0d5a9612f296646 in 1253ms, sequenceid=524, compaction requested=false at 1731715206441 (+5 ms)Writing region close event to WAL at 1731715206566 (+125 ms)Running coprocessor post-close hooks at 1731715206570 (+4 ms)Closed at 1731715206570 2024-11-16T00:00:06,572 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 6582600b266a5542c0d5a9612f296646 2024-11-16T00:00:06,573 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=6582600b266a5542c0d5a9612f296646, regionState=CLOSED 2024-11-16T00:00:06,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:06,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T00:00:06,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 6582600b266a5542c0d5a9612f296646, server=0801813e30ae,43013,1731715110098 in 1.5410 sec 2024-11-16T00:00:06,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-16T00:00:06,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6582600b266a5542c0d5a9612f296646, UNASSIGN in 1.5470 sec 2024-11-16T00:00:06,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T00:00:06,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5500 sec 2024-11-16T00:00:06,582 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715206582"}]},"ts":"1731715206582"} 2024-11-16T00:00:06,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T00:00:06,583 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T00:00:06,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5600 sec 2024-11-16T00:00:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T00:00:07,170 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:07,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-16T00:00:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,173 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T00:00:07,174 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,176 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646 2024-11-16T00:00:07,178 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/recovered.edits] 2024-11-16T00:00:07,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/1cfb5572ecf043dfbab98e99874031f3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/1cfb5572ecf043dfbab98e99874031f3 2024-11-16T00:00:07,183 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/e5de718bc0f74d11b8cfa18392f4380b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/A/e5de718bc0f74d11b8cfa18392f4380b 2024-11-16T00:00:07,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/7f87029b593c428586f9f342a48aab45 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/7f87029b593c428586f9f342a48aab45 2024-11-16T00:00:07,186 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d09d7d591f384554acc50cff229b2c88 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/B/d09d7d591f384554acc50cff229b2c88 2024-11-16T00:00:07,189 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/891bd5f4ab3345519dbec64b23250606 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/891bd5f4ab3345519dbec64b23250606 2024-11-16T00:00:07,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a9fe993087db4bd788a9b03f0b371aeb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/C/a9fe993087db4bd788a9b03f0b371aeb 2024-11-16T00:00:07,193 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/recovered.edits/527.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646/recovered.edits/527.seqid 2024-11-16T00:00:07,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/6582600b266a5542c0d5a9612f296646 2024-11-16T00:00:07,194 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T00:00:07,197 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,200 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T00:00:07,202 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T00:00:07,203 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,204 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T00:00:07,204 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715207204"}]},"ts":"9223372036854775807"} 2024-11-16T00:00:07,206 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T00:00:07,206 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6582600b266a5542c0d5a9612f296646, NAME => 'TestAcidGuarantees,,1731715176024.6582600b266a5542c0d5a9612f296646.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T00:00:07,206 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T00:00:07,206 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715207206"}]},"ts":"9223372036854775807"} 2024-11-16T00:00:07,207 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T00:00:07,208 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-16T00:00:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T00:00:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T00:00:07,284 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:07,292 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=235 (was 237), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 570) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4604 (was 5033) 2024-11-16T00:00:07,301 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=235, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=4603 2024-11-16T00:00:07,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T00:00:07,303 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T00:00:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:07,305 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T00:00:07,306 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:07,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-16T00:00:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:07,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T00:00:07,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742170_1346 (size=959) 2024-11-16T00:00:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:07,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-16T00:00:07,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742171_1347 (size=53) 2024-11-16T00:00:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:08,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:00:08,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9d50d9faa764c624311a6377059225df, disabling compactions & flushes 2024-11-16T00:00:08,122 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. after waiting 0 ms 2024-11-16T00:00:08,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,122 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,122 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d50d9faa764c624311a6377059225df: Waiting for close lock at 1731715208121Disabling compacts and flushes for region at 1731715208121Disabling writes for close at 1731715208122 (+1 ms)Writing region close event to WAL at 1731715208122Closed at 1731715208122 2024-11-16T00:00:08,123 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T00:00:08,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715208123"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715208123"}]},"ts":"1731715208123"} 2024-11-16T00:00:08,125 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T00:00:08,126 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T00:00:08,126 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715208126"}]},"ts":"1731715208126"} 2024-11-16T00:00:08,127 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T00:00:08,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, ASSIGN}] 2024-11-16T00:00:08,129 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, ASSIGN 2024-11-16T00:00:08,129 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-16T00:00:08,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:08,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, ASSIGN because future has completed 2024-11-16T00:00:08,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:08,437 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,438 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} 2024-11-16T00:00:08,438 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,438 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:00:08,438 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,438 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,440 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:08,441 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName A 2024-11-16T00:00:08,441 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:08,442 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:08,442 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,443 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName B 2024-11-16T00:00:08,443 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:08,444 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:08,444 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,445 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName C 2024-11-16T00:00:08,445 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:08,445 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:08,446 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,446 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,447 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,448 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,448 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,449 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T00:00:08,450 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,452 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T00:00:08,453 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 9d50d9faa764c624311a6377059225df; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60253880, jitterRate=-0.10214722156524658}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T00:00:08,453 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:08,454 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 9d50d9faa764c624311a6377059225df: Running coprocessor pre-open hook at 1731715208439Writing region info on filesystem at 1731715208439Initializing all the Stores at 1731715208439Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715208439Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715208439Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715208440 (+1 ms)Cleaning up temporary data from old regions at 1731715208448 (+8 ms)Running coprocessor post-open hooks at 1731715208453 (+5 ms)Region opened successfully at 1731715208454 (+1 ms) 2024-11-16T00:00:08,458 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., pid=85, masterSystemTime=1731715208434 2024-11-16T00:00:08,460 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,460 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:08,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:08,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:08,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T00:00:08,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 in 182 msec 2024-11-16T00:00:08,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-16T00:00:08,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, ASSIGN in 339 msec 2024-11-16T00:00:08,470 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T00:00:08,470 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715208470"}]},"ts":"1731715208470"} 2024-11-16T00:00:08,472 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T00:00:08,473 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T00:00:08,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-16T00:00:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-16T00:00:09,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:09,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22bc821c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:09,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:09,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:09,454 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24945c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:09,456 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a55ae6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:09,456 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:09,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:09,460 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T00:00:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-16T00:00:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23d435fb 2024-11-16T00:00:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T00:00:09,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T00:00:09,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T00:00:09,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T00:00:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:09,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742172_1348 (size=995) 2024-11-16T00:00:09,878 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T00:00:09,878 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T00:00:09,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:00:09,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE}] 2024-11-16T00:00:09,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE 2024-11-16T00:00:09,884 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:09,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE because future has completed 2024-11-16T00:00:09,886 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:00:09,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:10,038 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:00:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 9d50d9faa764c624311a6377059225df, disabling compactions & flushes 2024-11-16T00:00:10,038 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. after waiting 0 ms 2024-11-16T00:00:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,041 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T00:00:10,042 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,042 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 9d50d9faa764c624311a6377059225df: Waiting for close lock at 1731715210038Running coprocessor pre-close hooks at 1731715210038Disabling compacts and flushes for region at 1731715210038Disabling writes for close at 1731715210038Writing region close event to WAL at 1731715210038Running coprocessor post-close hooks at 1731715210042 (+4 ms)Closed at 1731715210042 2024-11-16T00:00:10,042 WARN [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 9d50d9faa764c624311a6377059225df to self. 2024-11-16T00:00:10,043 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,044 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=CLOSED 2024-11-16T00:00:10,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:10,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T00:00:10,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 in 160 msec 2024-11-16T00:00:10,048 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE; state=CLOSED, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=true 2024-11-16T00:00:10,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE because future has completed 2024-11-16T00:00:10,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:10,355 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,355 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} 2024-11-16T00:00:10,355 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,355 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:00:10,355 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,355 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,356 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,357 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName A 2024-11-16T00:00:10,358 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:10,358 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:10,359 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,359 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName B 2024-11-16T00:00:10,359 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:10,360 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:10,360 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,360 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d50d9faa764c624311a6377059225df columnFamilyName C 2024-11-16T00:00:10,360 DEBUG [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:10,360 INFO [StoreOpener-9d50d9faa764c624311a6377059225df-1 {}] regionserver.HStore(327): Store=9d50d9faa764c624311a6377059225df/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:10,361 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,361 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,362 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,363 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,363 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,363 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T00:00:10,364 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,365 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 9d50d9faa764c624311a6377059225df; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66767944, jitterRate=-0.005080103874206543}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T00:00:10,365 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,365 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 9d50d9faa764c624311a6377059225df: Running coprocessor pre-open hook at 1731715210355Writing region info on filesystem at 1731715210355Initializing all the Stores at 1731715210356 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715210356Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715210356Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715210356Cleaning up temporary data from old regions at 1731715210363 (+7 ms)Running coprocessor post-open hooks at 1731715210365 (+2 ms)Region opened successfully at 1731715210365 2024-11-16T00:00:10,366 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., pid=90, masterSystemTime=1731715210352 2024-11-16T00:00:10,367 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,367 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,368 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=OPEN, openSeqNum=5, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:10,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-16T00:00:10,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 in 170 msec 2024-11-16T00:00:10,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-16T00:00:10,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, REOPEN/MOVE in 488 msec 2024-11-16T00:00:10,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T00:00:10,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-16T00:00:10,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-16T00:00:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T00:00:10,377 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:10,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf842e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,378 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421ec4c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,380 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12068855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f539a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,382 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c47c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,383 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8b5dc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ef673d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,385 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc2ee0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,387 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483dc4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc7c0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,389 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f42f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,390 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc97b6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5958f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,392 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e647b90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,393 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73158c29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0f7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7676c5b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429208f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06cc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c68374d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2647f1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5990cd3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1ffabe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188a13f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8367c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e252a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625af565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df1ab09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:10,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:10,408 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6e888c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:10,410 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:10,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289cb948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:10,411 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,411 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,412 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:10,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,413 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-16T00:00:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:10,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:10,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:10,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:10,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,421 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,422 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,422 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,425 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,425 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,426 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,427 DEBUG [Thread-1588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,428 DEBUG [Thread-1588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:10,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:00:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,443 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:10,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T00:00:10,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5dfb1a88e2d4952b3ebb44e8dceac59_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715210442/Put/seqid=0 2024-11-16T00:00:10,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742173_1349 (size=14594) 2024-11-16T00:00:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:10,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715270521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715270531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715270545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715270550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715270551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715270659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715270659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715270659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715270670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715270670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:10,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:10,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:10,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c5dfb1a88e2d4952b3ebb44e8dceac59_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5dfb1a88e2d4952b3ebb44e8dceac59_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:10,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/4626ceb908aa4601b387d894b249613a, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:10,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/4626ceb908aa4601b387d894b249613a is 175, key is test_row_0/A:col10/1731715210442/Put/seqid=0 2024-11-16T00:00:10,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742174_1350 (size=39549) 2024-11-16T00:00:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715270880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715270880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715270880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715270880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715270880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:11,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:11,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715271200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715271200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715271200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715271200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715271200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/4626ceb908aa4601b387d894b249613a 2024-11-16T00:00:11,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/fe69556c932e4470b136faef2cd2c965 is 50, key is test_row_0/B:col10/1731715210442/Put/seqid=0 2024-11-16T00:00:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742175_1351 (size=12001) 2024-11-16T00:00:11,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/fe69556c932e4470b136faef2cd2c965 2024-11-16T00:00:11,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:11,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/61be4a9cf46e4caeac1a04c0f2480f6a is 50, key is test_row_0/C:col10/1731715210442/Put/seqid=0 2024-11-16T00:00:11,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742176_1352 (size=12001) 2024-11-16T00:00:11,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/61be4a9cf46e4caeac1a04c0f2480f6a 2024-11-16T00:00:11,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/4626ceb908aa4601b387d894b249613a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a 2024-11-16T00:00:11,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a, entries=200, sequenceid=16, filesize=38.6 K 2024-11-16T00:00:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/fe69556c932e4470b136faef2cd2c965 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965 2024-11-16T00:00:11,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T00:00:11,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/61be4a9cf46e4caeac1a04c0f2480f6a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a 2024-11-16T00:00:11,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T00:00:11,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9d50d9faa764c624311a6377059225df in 938ms, sequenceid=16, compaction requested=false 2024-11-16T00:00:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:11,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-16T00:00:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:11,495 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T00:00:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e2b67fbbf46b4fd2905d1ec36aa896ea_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715210530/Put/seqid=0 2024-11-16T00:00:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:11,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742177_1353 (size=12154) 2024-11-16T00:00:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:11,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:11,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715271730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715271734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715271738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715271738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715271738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715271850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715271860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715271860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715271861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715271861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:11,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:11,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:11,999 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e2b67fbbf46b4fd2905d1ec36aa896ea_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2b67fbbf46b4fd2905d1ec36aa896ea_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/5294c772226c470892b02987acb2166c, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/5294c772226c470892b02987acb2166c is 175, key is test_row_0/A:col10/1731715210530/Put/seqid=0 2024-11-16T00:00:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742178_1354 (size=30955) 2024-11-16T00:00:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715272070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715272070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715272071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715272071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715272071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,154 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T00:00:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715272391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,406 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/5294c772226c470892b02987acb2166c 2024-11-16T00:00:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715272391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715272391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715272391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715272391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/971bda632b564e96997d479905892b24 is 50, key is test_row_0/B:col10/1731715210530/Put/seqid=0 2024-11-16T00:00:12,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742179_1355 (size=12001) 2024-11-16T00:00:12,478 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/971bda632b564e96997d479905892b24 2024-11-16T00:00:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9d03411ebbdf40399d98c2dc82f5fefa is 50, key is test_row_0/C:col10/1731715210530/Put/seqid=0 2024-11-16T00:00:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742180_1356 (size=12001) 2024-11-16T00:00:12,525 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9d03411ebbdf40399d98c2dc82f5fefa 2024-11-16T00:00:12,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/5294c772226c470892b02987acb2166c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c 2024-11-16T00:00:12,539 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c, entries=150, sequenceid=41, filesize=30.2 K 2024-11-16T00:00:12,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/971bda632b564e96997d479905892b24 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24 2024-11-16T00:00:12,544 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T00:00:12,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9d03411ebbdf40399d98c2dc82f5fefa as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa 2024-11-16T00:00:12,552 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T00:00:12,553 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 9d50d9faa764c624311a6377059225df in 1058ms, sequenceid=41, compaction requested=false 2024-11-16T00:00:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-16T00:00:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-16T00:00:12,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-16T00:00:12,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1380 sec 2024-11-16T00:00:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:12,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1460 sec 2024-11-16T00:00:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:12,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T00:00:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:12,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f375c878f4e84c1db57fa4c021a34853_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:12,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742181_1357 (size=14594) 2024-11-16T00:00:12,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:12,961 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f375c878f4e84c1db57fa4c021a34853_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f375c878f4e84c1db57fa4c021a34853_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:12,964 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a258c6e9fcbf48d4ac0ce7fda54875a9, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:12,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a258c6e9fcbf48d4ac0ce7fda54875a9 is 175, key is test_row_0/A:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:12,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742182_1358 (size=39549) 2024-11-16T00:00:12,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a258c6e9fcbf48d4ac0ce7fda54875a9 2024-11-16T00:00:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715272965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715272976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715272974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715272985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:12,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/f2bda596895645459a338033f59cb8b0 is 50, key is test_row_0/B:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715272989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742183_1359 (size=12001) 2024-11-16T00:00:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715273100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715273100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715273101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715273101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715273109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715273310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715273310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715273310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715273310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715273320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/f2bda596895645459a338033f59cb8b0 2024-11-16T00:00:13,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/11802990c3674cb58adf143fed2efda1 is 50, key is test_row_0/C:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:13,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742184_1360 (size=12001) 2024-11-16T00:00:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715273630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715273630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715273630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715273631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715273639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:13,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/11802990c3674cb58adf143fed2efda1 2024-11-16T00:00:13,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a258c6e9fcbf48d4ac0ce7fda54875a9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9 2024-11-16T00:00:13,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9, entries=200, sequenceid=54, filesize=38.6 K 2024-11-16T00:00:13,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/f2bda596895645459a338033f59cb8b0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0 2024-11-16T00:00:13,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T00:00:13,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/11802990c3674cb58adf143fed2efda1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1 2024-11-16T00:00:13,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T00:00:13,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9d50d9faa764c624311a6377059225df in 994ms, sequenceid=54, compaction requested=true 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:13,910 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:13,910 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:13,911 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:13,911 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:13,911 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:13,911 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=107.5 K 2024-11-16T00:00:13,911 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:13,911 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9] 2024-11-16T00:00:13,911 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:13,911 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:13,912 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:13,912 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=35.2 K 2024-11-16T00:00:13,912 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4626ceb908aa4601b387d894b249613a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715210431 2024-11-16T00:00:13,912 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5294c772226c470892b02987acb2166c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731715210517 2024-11-16T00:00:13,912 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting fe69556c932e4470b136faef2cd2c965, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715210431 2024-11-16T00:00:13,912 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a258c6e9fcbf48d4ac0ce7fda54875a9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:13,912 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 971bda632b564e96997d479905892b24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731715210517 2024-11-16T00:00:13,913 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f2bda596895645459a338033f59cb8b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:13,928 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:13,931 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:13,932 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/7adb5e8d9b3843e98787f7e573b88dbf is 50, key is test_row_0/B:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:13,938 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411160f858c10dd2c49d681e4a22b48d35958_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:13,940 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411160f858c10dd2c49d681e4a22b48d35958_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:13,940 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160f858c10dd2c49d681e4a22b48d35958_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:13,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742185_1361 (size=12104) 2024-11-16T00:00:13,963 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/7adb5e8d9b3843e98787f7e573b88dbf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/7adb5e8d9b3843e98787f7e573b88dbf 2024-11-16T00:00:13,969 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into 7adb5e8d9b3843e98787f7e573b88dbf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:13,969 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:13,969 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=13, startTime=1731715213910; duration=0sec 2024-11-16T00:00:13,969 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:13,969 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:13,969 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:13,972 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:13,972 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:13,972 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:13,972 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=35.2 K 2024-11-16T00:00:13,973 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 61be4a9cf46e4caeac1a04c0f2480f6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715210431 2024-11-16T00:00:13,973 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d03411ebbdf40399d98c2dc82f5fefa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731715210517 2024-11-16T00:00:13,975 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 11802990c3674cb58adf143fed2efda1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:13,986 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:13,987 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f093484548774404bdb528c0d48bb17d is 50, key is test_row_0/C:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:13,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742186_1362 (size=4469) 2024-11-16T00:00:13,995 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#306 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:13,996 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6cf5d83760a24e818ec66980fe43301f is 175, key is test_row_0/A:col10/1731715211730/Put/seqid=0 2024-11-16T00:00:14,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742187_1363 (size=12104) 2024-11-16T00:00:14,008 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f093484548774404bdb528c0d48bb17d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f093484548774404bdb528c0d48bb17d 2024-11-16T00:00:14,015 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into f093484548774404bdb528c0d48bb17d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:14,015 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:14,015 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=13, startTime=1731715213910; duration=0sec 2024-11-16T00:00:14,015 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:14,015 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742188_1364 (size=31058) 2024-11-16T00:00:14,029 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6cf5d83760a24e818ec66980fe43301f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f 2024-11-16T00:00:14,039 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 6cf5d83760a24e818ec66980fe43301f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:14,039 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:14,039 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=13, startTime=1731715213910; duration=0sec 2024-11-16T00:00:14,039 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:14,039 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:14,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:14,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T00:00:14,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674283df748a74f0ea1cd52f580c29a5d_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715212974/Put/seqid=0 2024-11-16T00:00:14,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742189_1365 (size=17034) 2024-11-16T00:00:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715274174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715274174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715274175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715274178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715274189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715274300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715274300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715274300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715274300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715274312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715274510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715274510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715274510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715274510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715274530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-16T00:00:14,570 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:14,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-16T00:00:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:14,572 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:14,573 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:14,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:14,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:14,582 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674283df748a74f0ea1cd52f580c29a5d_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674283df748a74f0ea1cd52f580c29a5d_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:14,583 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/d5861d2c2f8f4863a80d4277a88c0e57, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:14,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/d5861d2c2f8f4863a80d4277a88c0e57 is 175, key is test_row_0/A:col10/1731715212974/Put/seqid=0 2024-11-16T00:00:14,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742190_1366 (size=48139) 2024-11-16T00:00:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:14,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:14,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715274829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715274830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715274830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715274830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715274849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:14,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:14,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:14,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:14,994 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/d5861d2c2f8f4863a80d4277a88c0e57 2024-11-16T00:00:15,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d74c4555268d43129aea819049c7a364 is 50, key is test_row_0/B:col10/1731715212974/Put/seqid=0 2024-11-16T00:00:15,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742191_1367 (size=12001) 2024-11-16T00:00:15,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:15,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715275340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715275349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715275350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715275350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:15,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715275360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:15,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:15,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d74c4555268d43129aea819049c7a364 2024-11-16T00:00:15,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f32dc840124747a1baef0afe87d6612a is 50, key is test_row_0/C:col10/1731715212974/Put/seqid=0 2024-11-16T00:00:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742192_1368 (size=12001) 2024-11-16T00:00:15,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:15,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:15,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f32dc840124747a1baef0afe87d6612a 2024-11-16T00:00:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/d5861d2c2f8f4863a80d4277a88c0e57 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57 2024-11-16T00:00:15,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57, entries=250, sequenceid=82, filesize=47.0 K 2024-11-16T00:00:15,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d74c4555268d43129aea819049c7a364 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364 2024-11-16T00:00:15,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T00:00:15,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f32dc840124747a1baef0afe87d6612a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a 2024-11-16T00:00:15,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T00:00:15,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9d50d9faa764c624311a6377059225df in 1708ms, sequenceid=82, compaction requested=false 2024-11-16T00:00:15,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:15,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T00:00:15,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:15,959 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T00:00:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160c38944a1496456889efea7bedfa6b22_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715214173/Put/seqid=0 2024-11-16T00:00:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742193_1369 (size=12154) 2024-11-16T00:00:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:16,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:16,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:16,384 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160c38944a1496456889efea7bedfa6b22_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c38944a1496456889efea7bedfa6b22_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:16,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/529dc6eb1c2347efbef5f7d363f10505, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:16,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/529dc6eb1c2347efbef5f7d363f10505 is 175, key is test_row_0/A:col10/1731715214173/Put/seqid=0 2024-11-16T00:00:16,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742194_1370 (size=30955) 2024-11-16T00:00:16,391 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/529dc6eb1c2347efbef5f7d363f10505 2024-11-16T00:00:16,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f0d36863e4d4c95891e5fbb022e4cb8 is 50, key is test_row_0/B:col10/1731715214173/Put/seqid=0 2024-11-16T00:00:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742195_1371 (size=12001) 2024-11-16T00:00:16,403 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f0d36863e4d4c95891e5fbb022e4cb8 2024-11-16T00:00:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/86be293bfdf7425fb963078b3add49ef is 50, key is test_row_0/C:col10/1731715214173/Put/seqid=0 2024-11-16T00:00:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742196_1372 (size=12001) 2024-11-16T00:00:16,440 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/86be293bfdf7425fb963078b3add49ef 2024-11-16T00:00:16,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715276421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715276423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715276424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/529dc6eb1c2347efbef5f7d363f10505 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505 2024-11-16T00:00:16,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715276433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,452 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505, entries=150, sequenceid=93, filesize=30.2 K 2024-11-16T00:00:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f0d36863e4d4c95891e5fbb022e4cb8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8 2024-11-16T00:00:16,457 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T00:00:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/86be293bfdf7425fb963078b3add49ef as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef 2024-11-16T00:00:16,463 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T00:00:16,465 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9d50d9faa764c624311a6377059225df in 506ms, sequenceid=93, compaction requested=true 2024-11-16T00:00:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-16T00:00:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-16T00:00:16,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T00:00:16,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8940 sec 2024-11-16T00:00:16,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8990 sec 2024-11-16T00:00:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:16,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:16,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T00:00:16,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669fc30b45cb54e2dbff5270b1391d246_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:16,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715276567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715276563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742197_1373 (size=19474) 2024-11-16T00:00:16,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715276575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111669fc30b45cb54e2dbff5270b1391d246_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669fc30b45cb54e2dbff5270b1391d246_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:16,585 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89e6797a67c142d5bfc44995e985223c, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:16,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89e6797a67c142d5bfc44995e985223c is 175, key is test_row_0/A:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:16,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715276576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715276577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742198_1374 (size=56733) 2024-11-16T00:00:16,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89e6797a67c142d5bfc44995e985223c 2024-11-16T00:00:16,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8579dd459908423fb7cf33e6226635ec is 50, key is test_row_0/B:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:16,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742199_1375 (size=12001) 2024-11-16T00:00:16,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715276680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715276680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715276690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715276699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715276699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T00:00:16,720 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:16,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T00:00:16,723 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:16,724 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:16,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:16,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T00:00:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:16,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715276899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715276899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715276899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715276910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715276910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:16,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:16,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8579dd459908423fb7cf33e6226635ec 2024-11-16T00:00:17,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T00:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:17,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/ab9899efe66444d8a56758c11ee5bac3 is 50, key is test_row_0/C:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:17,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742200_1376 (size=12001) 2024-11-16T00:00:17,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/ab9899efe66444d8a56758c11ee5bac3 2024-11-16T00:00:17,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89e6797a67c142d5bfc44995e985223c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c 2024-11-16T00:00:17,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c, entries=300, sequenceid=121, filesize=55.4 K 2024-11-16T00:00:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8579dd459908423fb7cf33e6226635ec as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec 2024-11-16T00:00:17,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec, entries=150, sequenceid=121, filesize=11.7 K 2024-11-16T00:00:17,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/ab9899efe66444d8a56758c11ee5bac3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3 2024-11-16T00:00:17,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3, entries=150, sequenceid=121, filesize=11.7 K 2024-11-16T00:00:17,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9d50d9faa764c624311a6377059225df in 573ms, sequenceid=121, compaction requested=true 2024-11-16T00:00:17,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:17,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:17,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:17,134 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:17,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:17,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:17,134 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:17,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:17,135 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:17,135 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:17,135 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:17,135 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:17,136 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,136 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,136 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=163.0 K 2024-11-16T00:00:17,136 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/7adb5e8d9b3843e98787f7e573b88dbf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=47.0 K 2024-11-16T00:00:17,136 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,136 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c] 2024-11-16T00:00:17,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:17,136 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7adb5e8d9b3843e98787f7e573b88dbf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:17,137 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf5d83760a24e818ec66980fe43301f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:17,137 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d74c4555268d43129aea819049c7a364, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731715212974 2024-11-16T00:00:17,140 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5861d2c2f8f4863a80d4277a88c0e57, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731715212962 2024-11-16T00:00:17,140 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0d36863e4d4c95891e5fbb022e4cb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731715214170 2024-11-16T00:00:17,140 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529dc6eb1c2347efbef5f7d363f10505, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731715214170 2024-11-16T00:00:17,140 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8579dd459908423fb7cf33e6226635ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216420 2024-11-16T00:00:17,141 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e6797a67c142d5bfc44995e985223c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216419 2024-11-16T00:00:17,175 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:17,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T00:00:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,190 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T00:00:17,192 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:17,193 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/bbb9b329e97746a1b500b14bd63d8f38 is 50, key is test_row_0/B:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:17,210 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411167b4f047740bd4765b3ba411cadc9a74c_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:17,213 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411167b4f047740bd4765b3ba411cadc9a74c_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:17,213 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167b4f047740bd4765b3ba411cadc9a74c_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111628a7f1ed92f542c6a056d374acb6d339_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715216562/Put/seqid=0 2024-11-16T00:00:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:17,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:17,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742202_1378 (size=4469) 2024-11-16T00:00:17,253 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#318 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:17,253 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/228aefedd8e94a608ba4b93651b79e49 is 175, key is test_row_0/A:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742201_1377 (size=12241) 2024-11-16T00:00:17,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742203_1379 (size=12154) 2024-11-16T00:00:17,271 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/bbb9b329e97746a1b500b14bd63d8f38 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bbb9b329e97746a1b500b14bd63d8f38 2024-11-16T00:00:17,278 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into bbb9b329e97746a1b500b14bd63d8f38(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:17,278 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:17,278 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=12, startTime=1731715217134; duration=0sec 2024-11-16T00:00:17,278 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:17,278 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:17,279 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:17,282 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:17,282 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:17,282 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:17,282 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f093484548774404bdb528c0d48bb17d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=47.0 K 2024-11-16T00:00:17,283 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f093484548774404bdb528c0d48bb17d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731715211730 2024-11-16T00:00:17,285 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f32dc840124747a1baef0afe87d6612a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731715212974 2024-11-16T00:00:17,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742204_1380 (size=31195) 2024-11-16T00:00:17,286 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 86be293bfdf7425fb963078b3add49ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731715214170 2024-11-16T00:00:17,286 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9899efe66444d8a56758c11ee5bac3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216420 2024-11-16T00:00:17,291 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/228aefedd8e94a608ba4b93651b79e49 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49 2024-11-16T00:00:17,296 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 228aefedd8e94a608ba4b93651b79e49(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:17,296 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:17,297 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=12, startTime=1731715217133; duration=0sec 2024-11-16T00:00:17,297 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:17,297 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:17,302 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:17,303 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/32c93536b3db4e93b277db6c0c352ba3 is 50, key is test_row_0/C:col10/1731715216420/Put/seqid=0 2024-11-16T00:00:17,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715277297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742205_1381 (size=12241) 2024-11-16T00:00:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715277307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715277309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715277311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,326 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/32c93536b3db4e93b277db6c0c352ba3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/32c93536b3db4e93b277db6c0c352ba3 2024-11-16T00:00:17,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715277314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,332 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into 32c93536b3db4e93b277db6c0c352ba3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:17,332 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:17,332 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=12, startTime=1731715217134; duration=0sec 2024-11-16T00:00:17,332 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:17,332 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715277419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715277430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715277430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715277430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715277430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715277642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715277642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715277642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715277643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715277646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111628a7f1ed92f542c6a056d374acb6d339_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628a7f1ed92f542c6a056d374acb6d339_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/ec1edcb77eb743eab2a2f16fc2b30501, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/ec1edcb77eb743eab2a2f16fc2b30501 is 175, key is test_row_0/A:col10/1731715216562/Put/seqid=0 2024-11-16T00:00:17,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742206_1382 (size=30955) 2024-11-16T00:00:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715277960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715277960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715277960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715277960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715277961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,079 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/ec1edcb77eb743eab2a2f16fc2b30501 2024-11-16T00:00:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/85be174929a445f6958fafe6b1ef32b4 is 50, key is test_row_0/B:col10/1731715216562/Put/seqid=0 2024-11-16T00:00:18,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742207_1383 (size=12001) 2024-11-16T00:00:18,095 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/85be174929a445f6958fafe6b1ef32b4 2024-11-16T00:00:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9a1c0af77dc04d0cab925d7b04543c10 is 50, key is test_row_0/C:col10/1731715216562/Put/seqid=0 2024-11-16T00:00:18,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742208_1384 (size=12001) 2024-11-16T00:00:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715278480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715278480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715278481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715278481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715278490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:18,520 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9a1c0af77dc04d0cab925d7b04543c10 2024-11-16T00:00:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/ec1edcb77eb743eab2a2f16fc2b30501 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501 2024-11-16T00:00:18,538 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501, entries=150, sequenceid=129, filesize=30.2 K 2024-11-16T00:00:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/85be174929a445f6958fafe6b1ef32b4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4 2024-11-16T00:00:18,560 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4, entries=150, sequenceid=129, filesize=11.7 K 2024-11-16T00:00:18,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/9a1c0af77dc04d0cab925d7b04543c10 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10 2024-11-16T00:00:18,582 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10, entries=150, sequenceid=129, filesize=11.7 K 2024-11-16T00:00:18,583 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 9d50d9faa764c624311a6377059225df in 1393ms, sequenceid=129, compaction requested=false 2024-11-16T00:00:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T00:00:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T00:00:18,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T00:00:18,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-16T00:00:18,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8690 sec 2024-11-16T00:00:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T00:00:18,880 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:18,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T00:00:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T00:00:18,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:18,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:18,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T00:00:19,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T00:00:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,038 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T00:00:19,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164af86b9e871440bbbe1db843d36a65b3_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715217311/Put/seqid=0 2024-11-16T00:00:19,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742209_1385 (size=12304) 2024-11-16T00:00:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,089 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164af86b9e871440bbbe1db843d36a65b3_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164af86b9e871440bbbe1db843d36a65b3_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/8bc3e539f8b84737a0b0837858b4a5e1, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:19,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/8bc3e539f8b84737a0b0837858b4a5e1 is 175, key is test_row_0/A:col10/1731715217311/Put/seqid=0 2024-11-16T00:00:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742210_1386 (size=31105) 2024-11-16T00:00:19,110 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/8bc3e539f8b84737a0b0837858b4a5e1 2024-11-16T00:00:19,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b2a6b57dce3c4931899a905357508b84 is 50, key is test_row_0/B:col10/1731715217311/Put/seqid=0 2024-11-16T00:00:19,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742211_1387 (size=12151) 2024-11-16T00:00:19,150 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b2a6b57dce3c4931899a905357508b84 2024-11-16T00:00:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/beb9d72f0c054dc0a80b954318661202 is 50, key is test_row_0/C:col10/1731715217311/Put/seqid=0 2024-11-16T00:00:19,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742212_1388 (size=12151) 2024-11-16T00:00:19,181 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/beb9d72f0c054dc0a80b954318661202 2024-11-16T00:00:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/8bc3e539f8b84737a0b0837858b4a5e1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1 2024-11-16T00:00:19,197 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1, entries=150, sequenceid=160, filesize=30.4 K 2024-11-16T00:00:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b2a6b57dce3c4931899a905357508b84 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84 2024-11-16T00:00:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T00:00:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,205 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84, entries=150, sequenceid=160, filesize=11.9 K 2024-11-16T00:00:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/beb9d72f0c054dc0a80b954318661202 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202 2024-11-16T00:00:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,223 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202, entries=150, sequenceid=160, filesize=11.9 K 2024-11-16T00:00:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,225 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 9d50d9faa764c624311a6377059225df in 186ms, sequenceid=160, compaction requested=true 2024-11-16T00:00:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T00:00:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T00:00:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T00:00:19,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 342 msec 2024-11-16T00:00:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 349 msec 2024-11-16T00:00:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T00:00:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,510 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:19,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T00:00:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,514 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:19,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T00:00:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111684c2ee387bff4254aea343dfad7da54a_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T00:00:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742214_1390 (size=27248) 2024-11-16T00:00:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,699 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,704 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111684c2ee387bff4254aea343dfad7da54a_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111684c2ee387bff4254aea343dfad7da54a_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,705 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c44d3739023945098b29bfbdf3281418, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c44d3739023945098b29bfbdf3281418 is 175, key is test_row_0/A:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742213_1389 (size=83035) 2024-11-16T00:00:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c44d3739023945098b29bfbdf3281418 2024-11-16T00:00:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/74893cd9dbae4d15827035fec45b3113 is 50, key is test_row_0/B:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:19,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742215_1391 (size=26495) 2024-11-16T00:00:19,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/74893cd9dbae4d15827035fec45b3113 2024-11-16T00:00:19,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/def78aeb33c34d9a8d1f899ab059ede0 is 50, key is test_row_0/C:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715279733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742216_1392 (size=12151) 2024-11-16T00:00:19,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715279751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715279751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715279752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715279752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T00:00:19,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:19,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:19,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715279859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715279880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715279880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715279880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715279880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:19,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T00:00:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:19,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715280079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715280101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715280101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715280101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715280102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T00:00:20,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:20,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/def78aeb33c34d9a8d1f899ab059ede0 2024-11-16T00:00:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c44d3739023945098b29bfbdf3281418 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418 2024-11-16T00:00:20,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418, entries=450, sequenceid=172, filesize=81.1 K 2024-11-16T00:00:20,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/74893cd9dbae4d15827035fec45b3113 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113 2024-11-16T00:00:20,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113, entries=450, sequenceid=172, filesize=25.9 K 2024-11-16T00:00:20,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/def78aeb33c34d9a8d1f899ab059ede0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0 2024-11-16T00:00:20,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T00:00:20,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9d50d9faa764c624311a6377059225df in 555ms, sequenceid=172, compaction requested=true 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:20,187 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:20,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62888 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:20,189 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,189 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,189 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bbb9b329e97746a1b500b14bd63d8f38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=61.4 K 2024-11-16T00:00:20,189 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=172.2 K 2024-11-16T00:00:20,189 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418] 2024-11-16T00:00:20,189 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb9b329e97746a1b500b14bd63d8f38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216420 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228aefedd8e94a608ba4b93651b79e49, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216420 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 85be174929a445f6958fafe6b1ef32b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731715216562 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a6b57dce3c4931899a905357508b84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731715217295 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1edcb77eb743eab2a2f16fc2b30501, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731715216562 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 74893cd9dbae4d15827035fec45b3113, keycount=450, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219597 2024-11-16T00:00:20,190 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc3e539f8b84737a0b0837858b4a5e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731715217295 2024-11-16T00:00:20,191 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44d3739023945098b29bfbdf3281418, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219597 2024-11-16T00:00:20,204 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:20,205 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b0e4ce38e6e94db29b931f91f6966837 is 50, key is test_row_0/B:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:20,210 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:20,221 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111688648fb13a1a4e3789d45a6f342f08bd_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:20,225 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111688648fb13a1a4e3789d45a6f342f08bd_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:20,225 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111688648fb13a1a4e3789d45a6f342f08bd_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:20,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742217_1393 (size=12527) 2024-11-16T00:00:20,246 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b0e4ce38e6e94db29b931f91f6966837 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0e4ce38e6e94db29b931f91f6966837 2024-11-16T00:00:20,253 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into b0e4ce38e6e94db29b931f91f6966837(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:20,253 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:20,253 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=12, startTime=1731715220187; duration=0sec 2024-11-16T00:00:20,254 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:20,254 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:20,254 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:20,255 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:20,255 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:20,255 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,255 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/32c93536b3db4e93b277db6c0c352ba3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=47.4 K 2024-11-16T00:00:20,255 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c93536b3db4e93b277db6c0c352ba3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731715216420 2024-11-16T00:00:20,258 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1c0af77dc04d0cab925d7b04543c10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731715216562 2024-11-16T00:00:20,259 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting beb9d72f0c054dc0a80b954318661202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731715217295 2024-11-16T00:00:20,259 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting def78aeb33c34d9a8d1f899ab059ede0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219602 2024-11-16T00:00:20,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742218_1394 (size=4469) 2024-11-16T00:00:20,263 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#331 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:20,264 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a08f22b9528c482586b8b6843eb64bfe is 175, key is test_row_0/A:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:20,282 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#332 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:20,282 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f7dca9eb2ea74bbfaadf1e3c40a14131 is 50, key is test_row_0/C:col10/1731715219602/Put/seqid=0 2024-11-16T00:00:20,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T00:00:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:20,283 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T00:00:20,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742219_1395 (size=31481) 2024-11-16T00:00:20,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1a79081f2414ce682d289e269f9a5c8_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715219748/Put/seqid=0 2024-11-16T00:00:20,316 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/a08f22b9528c482586b8b6843eb64bfe as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe 2024-11-16T00:00:20,321 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into a08f22b9528c482586b8b6843eb64bfe(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:20,321 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:20,321 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=12, startTime=1731715220187; duration=0sec 2024-11-16T00:00:20,321 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:20,322 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:20,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742221_1397 (size=12304) 2024-11-16T00:00:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:20,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742220_1396 (size=12527) 2024-11-16T00:00:20,328 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1a79081f2414ce682d289e269f9a5c8_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1a79081f2414ce682d289e269f9a5c8_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:20,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/702c6eb8cd35401c96cd8c94fc1fad24, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/702c6eb8cd35401c96cd8c94fc1fad24 is 175, key is test_row_0/A:col10/1731715219748/Put/seqid=0 2024-11-16T00:00:20,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742222_1398 (size=31105) 2024-11-16T00:00:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/702c6eb8cd35401c96cd8c94fc1fad24 2024-11-16T00:00:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d832bfd85cd5478696be9dd631a672f7 is 50, key is test_row_0/B:col10/1731715219748/Put/seqid=0 2024-11-16T00:00:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742223_1399 (size=12151) 2024-11-16T00:00:20,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715280428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715280429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715280430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715280432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715280441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715280550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715280550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715280550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715280551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715280560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:20,729 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/f7dca9eb2ea74bbfaadf1e3c40a14131 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f7dca9eb2ea74bbfaadf1e3c40a14131 2024-11-16T00:00:20,735 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into f7dca9eb2ea74bbfaadf1e3c40a14131(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:20,735 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:20,735 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=12, startTime=1731715220187; duration=0sec 2024-11-16T00:00:20,735 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:20,735 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d832bfd85cd5478696be9dd631a672f7 2024-11-16T00:00:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/e0b3056ee9574ed2bdb0ef77f176e88e is 50, key is test_row_0/C:col10/1731715219748/Put/seqid=0 2024-11-16T00:00:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715280770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715280770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715280770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715280773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:20,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742224_1400 (size=12151) 2024-11-16T00:00:20,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715280783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715281090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715281090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715281090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715281090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715281110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,189 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/e0b3056ee9574ed2bdb0ef77f176e88e 2024-11-16T00:00:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/702c6eb8cd35401c96cd8c94fc1fad24 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24 2024-11-16T00:00:21,199 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24, entries=150, sequenceid=198, filesize=30.4 K 2024-11-16T00:00:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d832bfd85cd5478696be9dd631a672f7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7 2024-11-16T00:00:21,203 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T00:00:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/e0b3056ee9574ed2bdb0ef77f176e88e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e 2024-11-16T00:00:21,207 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T00:00:21,208 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9d50d9faa764c624311a6377059225df in 925ms, sequenceid=198, compaction requested=false 2024-11-16T00:00:21,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T00:00:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T00:00:21,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T00:00:21,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6950 sec 2024-11-16T00:00:21,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7020 sec 2024-11-16T00:00:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:21,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:21,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T00:00:21,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f89d4f14f45409d8ce5df714c8dd39a_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742225_1401 (size=14794) 2024-11-16T00:00:21,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:21,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f89d4f14f45409d8ce5df714c8dd39a_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f89d4f14f45409d8ce5df714c8dd39a_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:21,641 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f4f49ebb68b54363bc0a8f8682cc2884, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:21,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f4f49ebb68b54363bc0a8f8682cc2884 is 175, key is test_row_0/A:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:21,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742226_1402 (size=39749) 2024-11-16T00:00:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T00:00:21,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:21,660 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T00:00:21,663 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:21,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:21,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715281665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715281665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715281667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715281679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715281790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715281790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715281790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715281790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715281799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:21,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T00:00:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:21,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T00:00:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:21,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:22,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715282000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715282009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715282010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715282010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715282010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f4f49ebb68b54363bc0a8f8682cc2884 2024-11-16T00:00:22,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/850bfade070c4b2284d3355e6e6e141d is 50, key is test_row_0/B:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:22,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742227_1403 (size=12151) 2024-11-16T00:00:22,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/850bfade070c4b2284d3355e6e6e141d 2024-11-16T00:00:22,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c81dc1cf913e4175a68ff2c4ef52c25e is 50, key is test_row_0/C:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:22,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T00:00:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:22,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742228_1404 (size=12151) 2024-11-16T00:00:22,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c81dc1cf913e4175a68ff2c4ef52c25e 2024-11-16T00:00:22,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f4f49ebb68b54363bc0a8f8682cc2884 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884 2024-11-16T00:00:22,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884, entries=200, sequenceid=212, filesize=38.8 K 2024-11-16T00:00:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/850bfade070c4b2284d3355e6e6e141d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d 2024-11-16T00:00:22,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-16T00:00:22,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c81dc1cf913e4175a68ff2c4ef52c25e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e 2024-11-16T00:00:22,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e, entries=150, sequenceid=212, filesize=11.9 K 2024-11-16T00:00:22,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9d50d9faa764c624311a6377059225df in 632ms, sequenceid=212, compaction requested=true 2024-11-16T00:00:22,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:22,247 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:22,248 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:22,249 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:22,249 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:22,249 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,249 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=99.9 K 2024-11-16T00:00:22,249 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,250 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884] 2024-11-16T00:00:22,250 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08f22b9528c482586b8b6843eb64bfe, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219602 2024-11-16T00:00:22,250 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:22,250 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702c6eb8cd35401c96cd8c94fc1fad24, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731715219748 2024-11-16T00:00:22,250 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:22,250 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,250 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0e4ce38e6e94db29b931f91f6966837, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.0 K 2024-11-16T00:00:22,251 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e4ce38e6e94db29b931f91f6966837, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219602 2024-11-16T00:00:22,251 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f49ebb68b54363bc0a8f8682cc2884, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220426 2024-11-16T00:00:22,251 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d832bfd85cd5478696be9dd631a672f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731715219748 2024-11-16T00:00:22,252 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 850bfade070c4b2284d3355e6e6e141d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220428 2024-11-16T00:00:22,266 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:22,280 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:22,280 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/1a1f7a46142c48369d57055cd7007053 is 50, key is test_row_0/B:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:22,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T00:00:22,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,282 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T00:00:22,286 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d62f8ded70814ed5864991ff35746bbe_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:22,287 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d62f8ded70814ed5864991ff35746bbe_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:22,287 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d62f8ded70814ed5864991ff35746bbe_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b5be480bc40a4033a6810d0022299f5e_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715221665/Put/seqid=0 2024-11-16T00:00:22,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742229_1405 (size=12629) 2024-11-16T00:00:22,300 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/1a1f7a46142c48369d57055cd7007053 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/1a1f7a46142c48369d57055cd7007053 2024-11-16T00:00:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742230_1406 (size=4469) 2024-11-16T00:00:22,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742231_1407 (size=12304) 2024-11-16T00:00:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:22,307 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into 1a1f7a46142c48369d57055cd7007053(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:22,307 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:22,307 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=13, startTime=1731715222248; duration=0sec 2024-11-16T00:00:22,307 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:22,307 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:22,307 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:22,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:22,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:22,312 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,312 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f7dca9eb2ea74bbfaadf1e3c40a14131, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.0 K 2024-11-16T00:00:22,312 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f7dca9eb2ea74bbfaadf1e3c40a14131, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715219602 2024-11-16T00:00:22,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b3056ee9574ed2bdb0ef77f176e88e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731715219748 2024-11-16T00:00:22,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c81dc1cf913e4175a68ff2c4ef52c25e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220428 2024-11-16T00:00:22,314 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b5be480bc40a4033a6810d0022299f5e_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b5be480bc40a4033a6810d0022299f5e_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/264ed40942f44ee1bd37b11b76cd2b6b, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/264ed40942f44ee1bd37b11b76cd2b6b is 175, key is test_row_0/A:col10/1731715221665/Put/seqid=0 2024-11-16T00:00:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:22,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742232_1408 (size=31105) 2024-11-16T00:00:22,331 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/264ed40942f44ee1bd37b11b76cd2b6b 2024-11-16T00:00:22,332 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#342 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:22,335 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/84c46e8c04444d6aba1829a888fd9c07 is 50, key is test_row_0/C:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:22,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f87168d6e3e4545933717e6aebebba1 is 50, key is test_row_0/B:col10/1731715221665/Put/seqid=0 2024-11-16T00:00:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715282347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715282348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715282350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742233_1409 (size=12629) 2024-11-16T00:00:22,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742234_1410 (size=12151) 2024-11-16T00:00:22,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715282363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715282358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,377 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f87168d6e3e4545933717e6aebebba1 2024-11-16T00:00:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/eb85a053ff18453b9539f7ff8fbc7ce3 is 50, key is test_row_0/C:col10/1731715221665/Put/seqid=0 2024-11-16T00:00:22,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742235_1411 (size=12151) 2024-11-16T00:00:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715282469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715282469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715282469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715282480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715282480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715282690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715282690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715282690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715282700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715282700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:22,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:22,706 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:22,707 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/df4b7d4cca2e46ecbc41c5d032aadab0 is 175, key is test_row_0/A:col10/1731715221612/Put/seqid=0 2024-11-16T00:00:22,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742236_1412 (size=31583) 2024-11-16T00:00:22,715 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/df4b7d4cca2e46ecbc41c5d032aadab0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0 2024-11-16T00:00:22,720 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into df4b7d4cca2e46ecbc41c5d032aadab0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:22,720 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:22,720 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=13, startTime=1731715222247; duration=0sec 2024-11-16T00:00:22,721 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:22,721 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:22,777 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/84c46e8c04444d6aba1829a888fd9c07 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/84c46e8c04444d6aba1829a888fd9c07 2024-11-16T00:00:22,783 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into 84c46e8c04444d6aba1829a888fd9c07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:22,783 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:22,783 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=13, startTime=1731715222249; duration=0sec 2024-11-16T00:00:22,783 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:22,783 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:22,809 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/eb85a053ff18453b9539f7ff8fbc7ce3 2024-11-16T00:00:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/264ed40942f44ee1bd37b11b76cd2b6b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b 2024-11-16T00:00:22,817 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b, entries=150, sequenceid=236, filesize=30.4 K 2024-11-16T00:00:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8f87168d6e3e4545933717e6aebebba1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1 2024-11-16T00:00:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T00:00:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/eb85a053ff18453b9539f7ff8fbc7ce3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3 2024-11-16T00:00:22,830 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3, entries=150, sequenceid=236, filesize=11.9 K 2024-11-16T00:00:22,831 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9d50d9faa764c624311a6377059225df in 549ms, sequenceid=236, compaction requested=false 2024-11-16T00:00:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T00:00:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T00:00:22,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T00:00:22,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1690 sec 2024-11-16T00:00:22,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1760 sec 2024-11-16T00:00:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:23,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T00:00:23,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162ecd961f889c40c490bd5689a94ed51b_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742237_1413 (size=14794) 2024-11-16T00:00:23,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:23,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162ecd961f889c40c490bd5689a94ed51b_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ecd961f889c40c490bd5689a94ed51b_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f3102f93a4264603b0ed169f24f77c41, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f3102f93a4264603b0ed169f24f77c41 is 175, key is test_row_0/A:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742238_1414 (size=39749) 2024-11-16T00:00:23,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f3102f93a4264603b0ed169f24f77c41 2024-11-16T00:00:23,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/bb8ae081cfbe400bab73f13162f057a3 is 50, key is test_row_0/B:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742239_1415 (size=12151) 2024-11-16T00:00:23,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/bb8ae081cfbe400bab73f13162f057a3 2024-11-16T00:00:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715283090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715283097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/af042469cd774013a4075a3fb0ba011e is 50, key is test_row_0/C:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715283099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715283106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715283106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742240_1416 (size=12151) 2024-11-16T00:00:23,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/af042469cd774013a4075a3fb0ba011e 2024-11-16T00:00:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/f3102f93a4264603b0ed169f24f77c41 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41 2024-11-16T00:00:23,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41, entries=200, sequenceid=252, filesize=38.8 K 2024-11-16T00:00:23,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/bb8ae081cfbe400bab73f13162f057a3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3 2024-11-16T00:00:23,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T00:00:23,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/af042469cd774013a4075a3fb0ba011e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e 2024-11-16T00:00:23,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T00:00:23,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 9d50d9faa764c624311a6377059225df in 143ms, sequenceid=252, compaction requested=true 2024-11-16T00:00:23,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:23,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:23,164 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:23,164 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:23,165 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:23,165 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:23,165 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,165 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=100.0 K 2024-11-16T00:00:23,165 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,165 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41] 2024-11-16T00:00:23,165 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:23,165 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:23,166 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,166 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/1a1f7a46142c48369d57055cd7007053, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.1 K 2024-11-16T00:00:23,167 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1f7a46142c48369d57055cd7007053, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220428 2024-11-16T00:00:23,167 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4b7d4cca2e46ecbc41c5d032aadab0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220428 2024-11-16T00:00:23,167 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f87168d6e3e4545933717e6aebebba1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731715221663 2024-11-16T00:00:23,168 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264ed40942f44ee1bd37b11b76cd2b6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731715221663 2024-11-16T00:00:23,168 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8ae081cfbe400bab73f13162f057a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222355 2024-11-16T00:00:23,169 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3102f93a4264603b0ed169f24f77c41, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222344 2024-11-16T00:00:23,198 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,203 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:23,204 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8102dd37cc284538bc82013a65f61047 is 50, key is test_row_0/B:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,209 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116763b5bfe4ed843a481e7899729ee0100_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,211 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116763b5bfe4ed843a481e7899729ee0100_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,211 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116763b5bfe4ed843a481e7899729ee0100_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,218 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:23,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T00:00:23,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742241_1417 (size=12731) 2024-11-16T00:00:23,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742242_1418 (size=4469) 2024-11-16T00:00:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715283249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715283252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,267 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#348 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:23,268 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89f9e82fb1ad4a458f2047c22014b680 is 175, key is test_row_0/A:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c159287367b44afea46d9f5d04f53f4d_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715223093/Put/seqid=0 2024-11-16T00:00:23,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715283258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715283266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715283267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742243_1419 (size=31685) 2024-11-16T00:00:23,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742244_1420 (size=14994) 2024-11-16T00:00:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715283369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715283369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715283380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715283390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715283390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715283590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715283590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715283599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715283610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715283610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:23,651 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/8102dd37cc284538bc82013a65f61047 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8102dd37cc284538bc82013a65f61047 2024-11-16T00:00:23,657 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into 8102dd37cc284538bc82013a65f61047(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:23,657 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:23,657 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=13, startTime=1731715223164; duration=0sec 2024-11-16T00:00:23,657 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:23,657 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:23,657 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:23,658 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:23,658 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:23,658 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,659 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/84c46e8c04444d6aba1829a888fd9c07, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.1 K 2024-11-16T00:00:23,659 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c46e8c04444d6aba1829a888fd9c07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731715220428 2024-11-16T00:00:23,659 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting eb85a053ff18453b9539f7ff8fbc7ce3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731715221663 2024-11-16T00:00:23,660 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting af042469cd774013a4075a3fb0ba011e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222355 2024-11-16T00:00:23,666 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:23,667 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/b331d82501254396ad5dab7ceae1e913 is 50, key is test_row_0/C:col10/1731715222355/Put/seqid=0 2024-11-16T00:00:23,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742245_1421 (size=12731) 2024-11-16T00:00:23,683 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/b331d82501254396ad5dab7ceae1e913 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/b331d82501254396ad5dab7ceae1e913 2024-11-16T00:00:23,687 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into b331d82501254396ad5dab7ceae1e913(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:23,687 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:23,687 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=13, startTime=1731715223164; duration=0sec 2024-11-16T00:00:23,687 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:23,687 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:23,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:23,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c159287367b44afea46d9f5d04f53f4d_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c159287367b44afea46d9f5d04f53f4d_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,732 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/89f9e82fb1ad4a458f2047c22014b680 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680 2024-11-16T00:00:23,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/1280210e0cd9422b843a7931655fce7d, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/1280210e0cd9422b843a7931655fce7d is 175, key is test_row_0/A:col10/1731715223093/Put/seqid=0 2024-11-16T00:00:23,742 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 89f9e82fb1ad4a458f2047c22014b680(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:23,743 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:23,743 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=13, startTime=1731715223163; duration=0sec 2024-11-16T00:00:23,743 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:23,743 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:23,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742246_1422 (size=39949) 2024-11-16T00:00:23,751 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/1280210e0cd9422b843a7931655fce7d 2024-11-16T00:00:23,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c6c68ec69bc64fb388ba27e9102f8c62 is 50, key is test_row_0/B:col10/1731715223093/Put/seqid=0 2024-11-16T00:00:23,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742247_1423 (size=12301) 2024-11-16T00:00:23,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c6c68ec69bc64fb388ba27e9102f8c62 2024-11-16T00:00:23,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d5d8a87114474338bf0bf2d90a959cce is 50, key is test_row_0/C:col10/1731715223093/Put/seqid=0 2024-11-16T00:00:23,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742248_1424 (size=12301) 2024-11-16T00:00:23,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d5d8a87114474338bf0bf2d90a959cce 2024-11-16T00:00:23,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/1280210e0cd9422b843a7931655fce7d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d 2024-11-16T00:00:23,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d, entries=200, sequenceid=277, filesize=39.0 K 2024-11-16T00:00:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T00:00:23,810 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:23,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:23,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c6c68ec69bc64fb388ba27e9102f8c62 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62 2024-11-16T00:00:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T00:00:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:23,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:23,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:23,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:23,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T00:00:23,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d5d8a87114474338bf0bf2d90a959cce as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce 2024-11-16T00:00:23,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T00:00:23,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 9d50d9faa764c624311a6377059225df in 606ms, sequenceid=277, compaction requested=false 2024-11-16T00:00:23,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:23,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T00:00:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:23,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bcbf85cde4be47258cbc6fcc630788fc_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:23,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742249_1425 (size=12454) 2024-11-16T00:00:23,949 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:23,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bcbf85cde4be47258cbc6fcc630788fc_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bcbf85cde4be47258cbc6fcc630788fc_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:23,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6dbb2ecd393d402e929347d9b1e3be8c, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:23,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6dbb2ecd393d402e929347d9b1e3be8c is 175, key is test_row_0/A:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:23,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742250_1426 (size=31255) 2024-11-16T00:00:23,967 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6dbb2ecd393d402e929347d9b1e3be8c 2024-11-16T00:00:23,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T00:00:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:23,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:23,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/e6244c08b3dc42a99d72df3db0577540 is 50, key is test_row_0/B:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742251_1427 (size=12301) 2024-11-16T00:00:24,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/e6244c08b3dc42a99d72df3db0577540 2024-11-16T00:00:24,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d6a50e73598e458b84d80147188ba423 is 50, key is test_row_0/C:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715283989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742252_1428 (size=12301) 2024-11-16T00:00:24,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d6a50e73598e458b84d80147188ba423 2024-11-16T00:00:24,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6dbb2ecd393d402e929347d9b1e3be8c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c 2024-11-16T00:00:24,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c, entries=150, sequenceid=294, filesize=30.5 K 2024-11-16T00:00:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/e6244c08b3dc42a99d72df3db0577540 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540 2024-11-16T00:00:24,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T00:00:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d6a50e73598e458b84d80147188ba423 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423 2024-11-16T00:00:24,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715284013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715284013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715284014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715284015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T00:00:24,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 9d50d9faa764c624311a6377059225df in 130ms, sequenceid=294, compaction requested=true 2024-11-16T00:00:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:24,045 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:24,046 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:24,046 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:24,046 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:24,047 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=100.5 K 2024-11-16T00:00:24,047 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:24,047 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c] 2024-11-16T00:00:24,047 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f9e82fb1ad4a458f2047c22014b680, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222355 2024-11-16T00:00:24,047 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1280210e0cd9422b843a7931655fce7d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731715223093 2024-11-16T00:00:24,048 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dbb2ecd393d402e929347d9b1e3be8c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:24,053 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:24,053 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:24,054 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:24,054 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:24,054 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8102dd37cc284538bc82013a65f61047, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.5 K 2024-11-16T00:00:24,054 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8102dd37cc284538bc82013a65f61047, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222355 2024-11-16T00:00:24,054 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c68ec69bc64fb388ba27e9102f8c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731715223093 2024-11-16T00:00:24,055 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e6244c08b3dc42a99d72df3db0577540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:24,067 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:24,081 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:24,082 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/6923d157d3ac4eb28612c09eaf4cd4b1 is 50, key is test_row_0/B:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:24,083 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d789d92bc9c545fb9bd9e44e9b676431_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:24,085 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d789d92bc9c545fb9bd9e44e9b676431_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:24,085 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d789d92bc9c545fb9bd9e44e9b676431_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:24,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742253_1429 (size=12983) 2024-11-16T00:00:24,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742254_1430 (size=4469) 2024-11-16T00:00:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:24,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T00:00:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:24,131 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T00:00:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:24,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:24,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0b975edfbd84f1b831746bf8735f016_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715224129/Put/seqid=0 2024-11-16T00:00:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742255_1431 (size=14994) 2024-11-16T00:00:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715284170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715284172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715284172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715284173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715284182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715284289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715284300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715284300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715284300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715284300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:24,516 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/6923d157d3ac4eb28612c09eaf4cd4b1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/6923d157d3ac4eb28612c09eaf4cd4b1 2024-11-16T00:00:24,520 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into 6923d157d3ac4eb28612c09eaf4cd4b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:24,520 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:24,520 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=13, startTime=1731715224053; duration=0sec 2024-11-16T00:00:24,520 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:24,520 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:24,520 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:24,521 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:24,521 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:24,521 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:24,522 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/b331d82501254396ad5dab7ceae1e913, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=36.5 K 2024-11-16T00:00:24,524 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b331d82501254396ad5dab7ceae1e913, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715222355 2024-11-16T00:00:24,524 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d8a87114474338bf0bf2d90a959cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731715223093 2024-11-16T00:00:24,525 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a50e73598e458b84d80147188ba423, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715284509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,530 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#357 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:24,531 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6998499921b34b7c8cd8d3933590ff04 is 175, key is test_row_0/A:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715284520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715284520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715284520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715284520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742256_1432 (size=31937) 2024-11-16T00:00:24,543 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:24,544 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/a3e378a97b654b568cf29e29219dfbfd is 50, key is test_row_0/C:col10/1731715223263/Put/seqid=0 2024-11-16T00:00:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742257_1433 (size=12983) 2024-11-16T00:00:24,565 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/a3e378a97b654b568cf29e29219dfbfd as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a3e378a97b654b568cf29e29219dfbfd 2024-11-16T00:00:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:24,573 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0b975edfbd84f1b831746bf8735f016_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0b975edfbd84f1b831746bf8735f016_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:24,573 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into a3e378a97b654b568cf29e29219dfbfd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:24,573 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:24,573 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=13, startTime=1731715224057; duration=0sec 2024-11-16T00:00:24,573 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:24,574 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c5a597e0c8e94344b5d40cd4ccd754b1, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:24,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c5a597e0c8e94344b5d40cd4ccd754b1 is 175, key is test_row_0/A:col10/1731715224129/Put/seqid=0 2024-11-16T00:00:24,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742258_1434 (size=39949) 2024-11-16T00:00:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715284830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715284839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715284839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715284839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715284840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:24,944 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6998499921b34b7c8cd8d3933590ff04 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04 2024-11-16T00:00:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:24,959 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 6998499921b34b7c8cd8d3933590ff04(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:24,959 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:24,959 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=13, startTime=1731715224045; duration=0sec 2024-11-16T00:00:24,959 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:24,959 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:24,985 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c5a597e0c8e94344b5d40cd4ccd754b1 2024-11-16T00:00:24,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c643b4ddd41244be946dde3922db7814 is 50, key is test_row_0/B:col10/1731715224129/Put/seqid=0 2024-11-16T00:00:25,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742259_1435 (size=12301) 2024-11-16T00:00:25,003 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c643b4ddd41244be946dde3922db7814 2024-11-16T00:00:25,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/7659314d36ad404aa7a28832f35b5d61 is 50, key is test_row_0/C:col10/1731715224129/Put/seqid=0 2024-11-16T00:00:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742260_1436 (size=12301) 2024-11-16T00:00:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715285350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715285359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715285359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715285360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715285360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:25,432 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/7659314d36ad404aa7a28832f35b5d61 2024-11-16T00:00:25,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/c5a597e0c8e94344b5d40cd4ccd754b1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1 2024-11-16T00:00:25,440 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1, entries=200, sequenceid=318, filesize=39.0 K 2024-11-16T00:00:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c643b4ddd41244be946dde3922db7814 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814 2024-11-16T00:00:25,444 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T00:00:25,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/7659314d36ad404aa7a28832f35b5d61 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61 2024-11-16T00:00:25,449 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T00:00:25,450 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9d50d9faa764c624311a6377059225df in 1318ms, sequenceid=318, compaction requested=false 2024-11-16T00:00:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T00:00:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T00:00:25,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T00:00:25,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6360 sec 2024-11-16T00:00:25,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6440 sec 2024-11-16T00:00:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T00:00:25,960 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:25,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T00:00:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T00:00:25,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:25,970 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:25,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T00:00:26,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T00:00:26,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:26,125 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T00:00:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0842cf2c4b443bcbcd813e4ae425971_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715224163/Put/seqid=0 2024-11-16T00:00:26,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742261_1437 (size=12454) 2024-11-16T00:00:26,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:26,240 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d0842cf2c4b443bcbcd813e4ae425971_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0842cf2c4b443bcbcd813e4ae425971_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/53110696e8e849809ab9079e11baa6d4, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/53110696e8e849809ab9079e11baa6d4 is 175, key is test_row_0/A:col10/1731715224163/Put/seqid=0 2024-11-16T00:00:26,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742262_1438 (size=31255) 2024-11-16T00:00:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T00:00:26,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:26,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715286445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715286448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715286449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715286459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715286460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715286570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715286570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715286570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715286570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715286581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T00:00:26,683 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/53110696e8e849809ab9079e11baa6d4 2024-11-16T00:00:26,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2128992b63541ba90fe65893f4a3014 is 50, key is test_row_0/B:col10/1731715224163/Put/seqid=0 2024-11-16T00:00:26,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742263_1439 (size=12301) 2024-11-16T00:00:26,701 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2128992b63541ba90fe65893f4a3014 2024-11-16T00:00:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/874d9e24f0b744de8a9b42baae66a5d4 is 50, key is test_row_0/C:col10/1731715224163/Put/seqid=0 2024-11-16T00:00:26,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742264_1440 (size=12301) 2024-11-16T00:00:26,740 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/874d9e24f0b744de8a9b42baae66a5d4 2024-11-16T00:00:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/53110696e8e849809ab9079e11baa6d4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4 2024-11-16T00:00:26,749 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4, entries=150, sequenceid=333, filesize=30.5 K 2024-11-16T00:00:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2128992b63541ba90fe65893f4a3014 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014 2024-11-16T00:00:26,753 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T00:00:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/874d9e24f0b744de8a9b42baae66a5d4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4 2024-11-16T00:00:26,767 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T00:00:26,768 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9d50d9faa764c624311a6377059225df in 643ms, sequenceid=333, compaction requested=true 2024-11-16T00:00:26,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:26,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:26,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T00:00:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T00:00:26,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T00:00:26,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 801 msec 2024-11-16T00:00:26,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 821 msec 2024-11-16T00:00:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:26,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:26,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T00:00:26,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111635070a5e544742b2a31e9e443b5d9c17_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715286804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715286819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715286819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715286820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715286821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742265_1441 (size=17534) 2024-11-16T00:00:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715286930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715286939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715286950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715286950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:26,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715286953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T00:00:27,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:27,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T00:00:27,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:27,116 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:27,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:27,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715287150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715287161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715287170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715287171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715287173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:27,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:27,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:27,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:27,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111635070a5e544742b2a31e9e443b5d9c17_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111635070a5e544742b2a31e9e443b5d9c17_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/b293f6b987fe4fc5bd8a5871e0b6e8de, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:27,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/b293f6b987fe4fc5bd8a5871e0b6e8de is 175, key is test_row_0/A:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:27,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742266_1442 (size=48639) 2024-11-16T00:00:27,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:27,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715287469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715287480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715287490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715287490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715287490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:27,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:27,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:27,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/b293f6b987fe4fc5bd8a5871e0b6e8de 2024-11-16T00:00:27,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c30cbd80824f4f7683fe66c1a0fe4d91 is 50, key is test_row_0/B:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742267_1443 (size=12301) 2024-11-16T00:00:27,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:27,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:27,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:27,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T00:00:27,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715287991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715288000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:28,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715288009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715288010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715288010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:28,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:28,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c30cbd80824f4f7683fe66c1a0fe4d91 2024-11-16T00:00:28,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/53982595e1f64c5799fde4a6ae858a5e is 50, key is test_row_0/C:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742268_1444 (size=12301) 2024-11-16T00:00:28,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/53982595e1f64c5799fde4a6ae858a5e 2024-11-16T00:00:28,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/b293f6b987fe4fc5bd8a5871e0b6e8de as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de 2024-11-16T00:00:28,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de, entries=250, sequenceid=360, filesize=47.5 K 2024-11-16T00:00:28,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/c30cbd80824f4f7683fe66c1a0fe4d91 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91 2024-11-16T00:00:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T00:00:28,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/53982595e1f64c5799fde4a6ae858a5e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e 2024-11-16T00:00:28,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T00:00:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 9d50d9faa764c624311a6377059225df in 1393ms, sequenceid=360, compaction requested=true 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:28,194 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T00:00:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,195 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:28,195 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:28,195 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,196 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=148.2 K 2024-11-16T00:00:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,196 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,196 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de] 2024-11-16T00:00:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,196 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6998499921b34b7c8cd8d3933590ff04, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,197 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a597e0c8e94344b5d40cd4ccd754b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731715223971 2024-11-16T00:00:28,197 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,197 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53110696e8e849809ab9079e11baa6d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715224163 2024-11-16T00:00:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,198 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b293f6b987fe4fc5bd8a5871e0b6e8de, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226447 2024-11-16T00:00:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:28,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:28,200 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,200 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/6923d157d3ac4eb28612c09eaf4cd4b1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=48.7 K 2024-11-16T00:00:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,201 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 6923d157d3ac4eb28612c09eaf4cd4b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T00:00:28,201 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c643b4ddd41244be946dde3922db7814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731715224010 2024-11-16T00:00:28,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,202 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T00:00:28,202 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a2128992b63541ba90fe65893f4a3014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715224163 2024-11-16T00:00:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,202 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c30cbd80824f4f7683fe66c1a0fe4d91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226457 2024-11-16T00:00:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,227 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,230 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,231 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/0cdccbf3fe664235ba5816243bf96396 is 50, key is test_row_0/B:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:28,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bd39253131eb4457b8bf369343c61671_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715226819/Put/seqid=0 2024-11-16T00:00:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,236 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ae9f10771015498992c2626f2563b009_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,239 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ae9f10771015498992c2626f2563b009_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:28,239 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae9f10771015498992c2626f2563b009_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742269_1445 (size=13119) 2024-11-16T00:00:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742271_1447 (size=4469) 2024-11-16T00:00:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742270_1446 (size=9914) 2024-11-16T00:00:28,257 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#369 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,258 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/88d79e733deb4d71b1b685245b972f14 is 175, key is test_row_0/A:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:28,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,270 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bd39253131eb4457b8bf369343c61671_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd39253131eb4457b8bf369343c61671_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/9ffd2e7a9b5d4d938e6d67b3721d55f3, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 is 175, key is test_row_0/A:col10/1731715226819/Put/seqid=0 2024-11-16T00:00:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742272_1448 (size=32073) 2024-11-16T00:00:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,287 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/88d79e733deb4d71b1b685245b972f14 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14 2024-11-16T00:00:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742273_1449 (size=22561) 2024-11-16T00:00:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,292 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 2024-11-16T00:00:28,292 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 88d79e733deb4d71b1b685245b972f14(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:28,292 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:28,292 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=12, startTime=1731715228194; duration=0sec 2024-11-16T00:00:28,292 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:28,292 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:28,292 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,298 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:28,298 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:28,298 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,298 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a3e378a97b654b568cf29e29219dfbfd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=48.7 K 2024-11-16T00:00:28,299 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e378a97b654b568cf29e29219dfbfd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731715223263 2024-11-16T00:00:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,300 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7659314d36ad404aa7a28832f35b5d61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731715224010 2024-11-16T00:00:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 874d9e24f0b744de8a9b42baae66a5d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715224163 2024-11-16T00:00:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,301 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53982595e1f64c5799fde4a6ae858a5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226457 2024-11-16T00:00:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/282f818b87e243339e0e64ef758278aa is 50, key is test_row_0/B:col10/1731715226819/Put/seqid=0 2024-11-16T00:00:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,316 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,316 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/753df2f526ad49858fec9860766000aa is 50, key is test_row_0/C:col10/1731715226797/Put/seqid=0 2024-11-16T00:00:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742274_1450 (size=9857) 2024-11-16T00:00:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742275_1451 (size=13119) 2024-11-16T00:00:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,373 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/753df2f526ad49858fec9860766000aa as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/753df2f526ad49858fec9860766000aa 2024-11-16T00:00:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,378 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into 753df2f526ad49858fec9860766000aa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:28,378 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:28,378 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=12, startTime=1731715228194; duration=0sec 2024-11-16T00:00:28,378 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:28,378 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,648 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/0cdccbf3fe664235ba5816243bf96396 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/0cdccbf3fe664235ba5816243bf96396 2024-11-16T00:00:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,654 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into 0cdccbf3fe664235ba5816243bf96396(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:28,654 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:28,654 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=12, startTime=1731715228194; duration=0sec 2024-11-16T00:00:28,654 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:28,654 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,727 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/282f818b87e243339e0e64ef758278aa 2024-11-16T00:00:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/a22c7646765a48d7bb1670b3363b12b4 is 50, key is test_row_0/C:col10/1731715226819/Put/seqid=0 2024-11-16T00:00:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742276_1452 (size=9857) 2024-11-16T00:00:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,769 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/a22c7646765a48d7bb1670b3363b12b4 2024-11-16T00:00:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 2024-11-16T00:00:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,779 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3, entries=100, sequenceid=370, filesize=22.0 K 2024-11-16T00:00:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/282f818b87e243339e0e64ef758278aa as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa 2024-11-16T00:00:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,785 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa, entries=100, sequenceid=370, filesize=9.6 K 2024-11-16T00:00:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/a22c7646765a48d7bb1670b3363b12b4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4 2024-11-16T00:00:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,791 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4, entries=100, sequenceid=370, filesize=9.6 K 2024-11-16T00:00:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,792 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 9d50d9faa764c624311a6377059225df in 590ms, sequenceid=370, compaction requested=false 2024-11-16T00:00:28,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:28,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T00:00:28,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-16T00:00:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.6870 sec 2024-11-16T00:00:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:29,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T00:00:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116267fc80dc17b4ca09345b5545918d42c_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742278_1454 (size=25158) 2024-11-16T00:00:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116267fc80dc17b4ca09345b5545918d42c_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116267fc80dc17b4ca09345b5545918d42c_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6963379513c84f09b83ad7b237ef0654, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6963379513c84f09b83ad7b237ef0654 is 175, key is test_row_0/A:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742277_1453 (size=74795) 2024-11-16T00:00:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715289209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715289214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715289216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715289218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715289220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T00:00:29,262 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:29,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-16T00:00:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:29,265 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:29,266 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:29,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715289329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715289340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715289340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715289350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715289353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:29,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T00:00:29,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:29,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6963379513c84f09b83ad7b237ef0654 2024-11-16T00:00:29,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/3df271ec6be440919176f75d9ff50f19 is 50, key is test_row_0/B:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715289550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742279_1455 (size=12301) 2024-11-16T00:00:29,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/3df271ec6be440919176f75d9ff50f19 2024-11-16T00:00:29,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715289560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715289560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d1b1885321614139ab1232c5bb27d672 is 50, key is test_row_0/C:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T00:00:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:29,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742280_1456 (size=12301) 2024-11-16T00:00:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715289570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715289571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d1b1885321614139ab1232c5bb27d672 2024-11-16T00:00:29,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/6963379513c84f09b83ad7b237ef0654 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654 2024-11-16T00:00:29,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654, entries=400, sequenceid=384, filesize=73.0 K 2024-11-16T00:00:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/3df271ec6be440919176f75d9ff50f19 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19 2024-11-16T00:00:29,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19, entries=150, sequenceid=384, filesize=12.0 K 2024-11-16T00:00:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d1b1885321614139ab1232c5bb27d672 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672 2024-11-16T00:00:29,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672, entries=150, sequenceid=384, filesize=12.0 K 2024-11-16T00:00:29,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9d50d9faa764c624311a6377059225df in 519ms, sequenceid=384, compaction requested=true 2024-11-16T00:00:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:29,608 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:29,608 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:29,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:29,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:29,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:29,609 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129429 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:29,609 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/A is initiating minor compaction (all files) 2024-11-16T00:00:29,610 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/A in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,610 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=126.4 K 2024-11-16T00:00:29,610 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,610 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654] 2024-11-16T00:00:29,610 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:29,610 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/B is initiating minor compaction (all files) 2024-11-16T00:00:29,610 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/B in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,610 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/0cdccbf3fe664235ba5816243bf96396, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=34.5 K 2024-11-16T00:00:29,611 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cdccbf3fe664235ba5816243bf96396, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226457 2024-11-16T00:00:29,611 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d79e733deb4d71b1b685245b972f14, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226457 2024-11-16T00:00:29,611 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffd2e7a9b5d4d938e6d67b3721d55f3, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731715226818 2024-11-16T00:00:29,611 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 282f818b87e243339e0e64ef758278aa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731715226818 2024-11-16T00:00:29,620 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3df271ec6be440919176f75d9ff50f19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715229084 2024-11-16T00:00:29,627 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6963379513c84f09b83ad7b237ef0654, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715229064 2024-11-16T00:00:29,640 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:29,640 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d74e0d7357604fcea2b10c665ac55555 is 50, key is test_row_0/B:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,642 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,646 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411161d2650541bd8480cbe0df0a5d63c69a3_9d50d9faa764c624311a6377059225df store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,648 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411161d2650541bd8480cbe0df0a5d63c69a3_9d50d9faa764c624311a6377059225df, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,649 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161d2650541bd8480cbe0df0a5d63c69a3_9d50d9faa764c624311a6377059225df because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742281_1457 (size=13221) 2024-11-16T00:00:29,664 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/d74e0d7357604fcea2b10c665ac55555 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74e0d7357604fcea2b10c665ac55555 2024-11-16T00:00:29,671 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/B of 9d50d9faa764c624311a6377059225df into d74e0d7357604fcea2b10c665ac55555(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:29,671 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:29,671 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/B, priority=13, startTime=1731715229608; duration=0sec 2024-11-16T00:00:29,671 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:29,671 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:29,671 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:29,672 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:29,672 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 9d50d9faa764c624311a6377059225df/C is initiating minor compaction (all files) 2024-11-16T00:00:29,672 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d50d9faa764c624311a6377059225df/C in TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,673 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/753df2f526ad49858fec9860766000aa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp, totalSize=34.5 K 2024-11-16T00:00:29,673 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 753df2f526ad49858fec9860766000aa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731715226457 2024-11-16T00:00:29,673 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a22c7646765a48d7bb1670b3363b12b4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731715226818 2024-11-16T00:00:29,674 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b1885321614139ab1232c5bb27d672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731715229084 2024-11-16T00:00:29,695 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:29,696 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d808129a8f65443c96ee54216eb19324 is 50, key is test_row_0/C:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742282_1458 (size=4469) 2024-11-16T00:00:29,706 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d50d9faa764c624311a6377059225df#A#compaction#379 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:29,707 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/0f8ed03684c7490d9f903618c16120fc is 175, key is test_row_0/A:col10/1731715229087/Put/seqid=0 2024-11-16T00:00:29,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742283_1459 (size=13221) 2024-11-16T00:00:29,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T00:00:29,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:29,728 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T00:00:29,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742284_1460 (size=32175) 2024-11-16T00:00:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111683b6492b9d52481fb4d9fca456c25711_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715229206/Put/seqid=0 2024-11-16T00:00:29,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742285_1461 (size=12454) 2024-11-16T00:00:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:29,772 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111683b6492b9d52481fb4d9fca456c25711_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111683b6492b9d52481fb4d9fca456c25711_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/7099aec9b7e44466b6f4c0c7d213f36b, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/7099aec9b7e44466b6f4c0c7d213f36b is 175, key is test_row_0/A:col10/1731715229206/Put/seqid=0 2024-11-16T00:00:29,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742286_1462 (size=31255) 2024-11-16T00:00:29,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. as already flushing 2024-11-16T00:00:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715289895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715289896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715289897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715289898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715289907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715290010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,119 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/d808129a8f65443c96ee54216eb19324 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d808129a8f65443c96ee54216eb19324 2024-11-16T00:00:30,126 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/C of 9d50d9faa764c624311a6377059225df into d808129a8f65443c96ee54216eb19324(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:30,126 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:30,126 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/C, priority=13, startTime=1731715229609; duration=0sec 2024-11-16T00:00:30,126 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:30,126 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:30,136 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/0f8ed03684c7490d9f903618c16120fc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/0f8ed03684c7490d9f903618c16120fc 2024-11-16T00:00:30,141 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d50d9faa764c624311a6377059225df/A of 9d50d9faa764c624311a6377059225df into 0f8ed03684c7490d9f903618c16120fc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:30,141 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:30,141 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., storeName=9d50d9faa764c624311a6377059225df/A, priority=13, startTime=1731715229608; duration=0sec 2024-11-16T00:00:30,141 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:30,141 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:30,202 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/7099aec9b7e44466b6f4c0c7d213f36b 2024-11-16T00:00:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2dfe9753512436ab0f8323ef8c347e4 is 50, key is test_row_0/B:col10/1731715229206/Put/seqid=0 2024-11-16T00:00:30,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742287_1463 (size=12301) 2024-11-16T00:00:30,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715290230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715290240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715290241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715290241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715290241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:30,434 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-16T00:00:30,434 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:30,435 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,435 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-16T00:00:30,435 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,435 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:30,435 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:30,435 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:30,436 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-16T00:00:30,437 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:30,437 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,437 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:30,444 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-16T00:00:30,444 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:30,444 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,444 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:30,446 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-16T00:00:30,446 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:30,446 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,446 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:30,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33208 deadline: 1731715290550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33244 deadline: 1731715290550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33264 deadline: 1731715290550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33228 deadline: 1731715290560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33280 deadline: 1731715290560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:00:30,630 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2dfe9753512436ab0f8323ef8c347e4 2024-11-16T00:00:30,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5f5e6da435b408ebc1df822e9a8d1b7 is 50, key is test_row_0/C:col10/1731715229206/Put/seqid=0 2024-11-16T00:00:30,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742288_1464 (size=12301) 2024-11-16T00:00:31,042 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5f5e6da435b408ebc1df822e9a8d1b7 2024-11-16T00:00:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/7099aec9b7e44466b6f4c0c7d213f36b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/7099aec9b7e44466b6f4c0c7d213f36b 2024-11-16T00:00:31,048 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/7099aec9b7e44466b6f4c0c7d213f36b, entries=150, sequenceid=410, filesize=30.5 K 2024-11-16T00:00:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/a2dfe9753512436ab0f8323ef8c347e4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2dfe9753512436ab0f8323ef8c347e4 2024-11-16T00:00:31,053 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2dfe9753512436ab0f8323ef8c347e4, entries=150, sequenceid=410, filesize=12.0 K 2024-11-16T00:00:31,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5f5e6da435b408ebc1df822e9a8d1b7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5f5e6da435b408ebc1df822e9a8d1b7 2024-11-16T00:00:31,058 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5f5e6da435b408ebc1df822e9a8d1b7, entries=150, sequenceid=410, filesize=12.0 K 2024-11-16T00:00:31,059 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9d50d9faa764c624311a6377059225df in 1331ms, sequenceid=410, compaction requested=false 2024-11-16T00:00:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-16T00:00:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-16T00:00:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:31,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T00:00:31,063 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-16T00:00:31,063 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:31,063 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T00:00:31,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7950 sec 2024-11-16T00:00:31,063 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,063 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-16T00:00:31,063 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:31,064 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,064 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,065 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-16T00:00:31,065 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:31,065 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,065 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8000 sec 2024-11-16T00:00:31,071 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-16T00:00:31,071 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:31,071 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,071 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,072 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-16T00:00:31,072 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:31,072 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,072 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116adf20bc60fcc4bc88b89b7e3acee7c5e_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715231060/Put/seqid=0 2024-11-16T00:00:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742289_1465 (size=12454) 2024-11-16T00:00:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T00:00:31,420 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 963 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2889 rows 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2883 rows 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 978 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2920 rows 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 964 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2883 rows 2024-11-16T00:00:31,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 970 2024-11-16T00:00:31,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2908 rows 2024-11-16T00:00:31,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:00:31,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:00:31,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:31,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:31,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-16T00:00:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:31,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715231425"}]},"ts":"1731715231425"} 2024-11-16T00:00:31,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T00:00:31,427 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T00:00:31,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:00:31,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, UNASSIGN}] 2024-11-16T00:00:31,430 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, UNASSIGN 2024-11-16T00:00:31,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:31,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, UNASSIGN because future has completed 2024-11-16T00:00:31,432 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:00:31,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:31,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:31,490 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116adf20bc60fcc4bc88b89b7e3acee7c5e_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116adf20bc60fcc4bc88b89b7e3acee7c5e_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:31,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e7f1982209e64590a3b4e985975daf3e, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:31,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e7f1982209e64590a3b4e985975daf3e is 175, key is test_row_0/A:col10/1731715231060/Put/seqid=0 2024-11-16T00:00:31,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742290_1466 (size=31255) 2024-11-16T00:00:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:31,585 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:31,585 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:00:31,585 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 9d50d9faa764c624311a6377059225df, disabling compactions & flushes 2024-11-16T00:00:31,585 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:31,895 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e7f1982209e64590a3b4e985975daf3e 2024-11-16T00:00:31,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b0d7f17564db48aba49ecc7796697a30 is 50, key is test_row_0/B:col10/1731715231060/Put/seqid=0 2024-11-16T00:00:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742291_1467 (size=12301) 2024-11-16T00:00:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:32,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b0d7f17564db48aba49ecc7796697a30 2024-11-16T00:00:32,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/3457e66d622443399e8adf0ffaf1abf0 is 50, key is test_row_0/C:col10/1731715231060/Put/seqid=0 2024-11-16T00:00:32,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742292_1468 (size=12301) 2024-11-16T00:00:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:32,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/3457e66d622443399e8adf0ffaf1abf0 2024-11-16T00:00:32,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e7f1982209e64590a3b4e985975daf3e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e7f1982209e64590a3b4e985975daf3e 2024-11-16T00:00:32,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e7f1982209e64590a3b4e985975daf3e, entries=150, sequenceid=426, filesize=30.5 K 2024-11-16T00:00:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/b0d7f17564db48aba49ecc7796697a30 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0d7f17564db48aba49ecc7796697a30 2024-11-16T00:00:32,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0d7f17564db48aba49ecc7796697a30, entries=150, sequenceid=426, filesize=12.0 K 2024-11-16T00:00:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/3457e66d622443399e8adf0ffaf1abf0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/3457e66d622443399e8adf0ffaf1abf0 2024-11-16T00:00:32,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/3457e66d622443399e8adf0ffaf1abf0, entries=150, sequenceid=426, filesize=12.0 K 2024-11-16T00:00:32,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for 9d50d9faa764c624311a6377059225df in 1670ms, sequenceid=426, compaction requested=true 2024-11-16T00:00:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d50d9faa764c624311a6377059225df: 2024-11-16T00:00:32,732 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:32,733 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:32,733 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. after waiting 0 ms 2024-11-16T00:00:32,733 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d50d9faa764c624311a6377059225df:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:32,733 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 9d50d9faa764c624311a6377059225df 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-16T00:00:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. because compaction request was cancelled 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:A 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. because compaction request was cancelled 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. because compaction request was cancelled 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:C 2024-11-16T00:00:32,733 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d50d9faa764c624311a6377059225df:B 2024-11-16T00:00:32,737 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674276463ea274a79856955a8c226e534_9d50d9faa764c624311a6377059225df is 50, key is test_row_0/A:col10/1731715231070/Put/seqid=0 2024-11-16T00:00:32,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742293_1469 (size=9914) 2024-11-16T00:00:32,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654] to archive 2024-11-16T00:00:32,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:32,862 WARN [regionserver/0801813e30ae:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-16T00:00:32,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/4626ceb908aa4601b387d894b249613a 2024-11-16T00:00:32,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/5294c772226c470892b02987acb2166c 2024-11-16T00:00:32,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a258c6e9fcbf48d4ac0ce7fda54875a9 2024-11-16T00:00:32,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6cf5d83760a24e818ec66980fe43301f 2024-11-16T00:00:32,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/d5861d2c2f8f4863a80d4277a88c0e57 2024-11-16T00:00:32,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/529dc6eb1c2347efbef5f7d363f10505 2024-11-16T00:00:32,874 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89e6797a67c142d5bfc44995e985223c 2024-11-16T00:00:32,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/228aefedd8e94a608ba4b93651b79e49 2024-11-16T00:00:32,876 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/ec1edcb77eb743eab2a2f16fc2b30501 2024-11-16T00:00:32,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/8bc3e539f8b84737a0b0837858b4a5e1 2024-11-16T00:00:32,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c44d3739023945098b29bfbdf3281418 2024-11-16T00:00:32,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/a08f22b9528c482586b8b6843eb64bfe 2024-11-16T00:00:32,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/702c6eb8cd35401c96cd8c94fc1fad24 2024-11-16T00:00:32,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f4f49ebb68b54363bc0a8f8682cc2884 2024-11-16T00:00:32,883 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/df4b7d4cca2e46ecbc41c5d032aadab0 2024-11-16T00:00:32,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/264ed40942f44ee1bd37b11b76cd2b6b 2024-11-16T00:00:32,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/f3102f93a4264603b0ed169f24f77c41 2024-11-16T00:00:32,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/89f9e82fb1ad4a458f2047c22014b680 2024-11-16T00:00:32,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/1280210e0cd9422b843a7931655fce7d 2024-11-16T00:00:32,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6998499921b34b7c8cd8d3933590ff04 2024-11-16T00:00:32,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6dbb2ecd393d402e929347d9b1e3be8c 2024-11-16T00:00:32,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/c5a597e0c8e94344b5d40cd4ccd754b1 2024-11-16T00:00:32,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/53110696e8e849809ab9079e11baa6d4 2024-11-16T00:00:32,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/b293f6b987fe4fc5bd8a5871e0b6e8de 2024-11-16T00:00:32,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/88d79e733deb4d71b1b685245b972f14 2024-11-16T00:00:32,894 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/9ffd2e7a9b5d4d938e6d67b3721d55f3 2024-11-16T00:00:32,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/6963379513c84f09b83ad7b237ef0654 2024-11-16T00:00:32,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/7adb5e8d9b3843e98787f7e573b88dbf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bbb9b329e97746a1b500b14bd63d8f38, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0e4ce38e6e94db29b931f91f6966837, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/1a1f7a46142c48369d57055cd7007053, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8102dd37cc284538bc82013a65f61047, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/6923d157d3ac4eb28612c09eaf4cd4b1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/0cdccbf3fe664235ba5816243bf96396, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19] to archive 2024-11-16T00:00:32,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:32,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/fe69556c932e4470b136faef2cd2c965 2024-11-16T00:00:32,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/971bda632b564e96997d479905892b24 2024-11-16T00:00:32,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/7adb5e8d9b3843e98787f7e573b88dbf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/7adb5e8d9b3843e98787f7e573b88dbf 2024-11-16T00:00:32,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/f2bda596895645459a338033f59cb8b0 2024-11-16T00:00:32,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74c4555268d43129aea819049c7a364 2024-11-16T00:00:32,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f0d36863e4d4c95891e5fbb022e4cb8 2024-11-16T00:00:32,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bbb9b329e97746a1b500b14bd63d8f38 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bbb9b329e97746a1b500b14bd63d8f38 2024-11-16T00:00:32,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8579dd459908423fb7cf33e6226635ec 2024-11-16T00:00:32,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/85be174929a445f6958fafe6b1ef32b4 2024-11-16T00:00:32,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b2a6b57dce3c4931899a905357508b84 2024-11-16T00:00:32,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/74893cd9dbae4d15827035fec45b3113 2024-11-16T00:00:32,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0e4ce38e6e94db29b931f91f6966837 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0e4ce38e6e94db29b931f91f6966837 2024-11-16T00:00:32,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d832bfd85cd5478696be9dd631a672f7 2024-11-16T00:00:32,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/1a1f7a46142c48369d57055cd7007053 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/1a1f7a46142c48369d57055cd7007053 2024-11-16T00:00:32,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/850bfade070c4b2284d3355e6e6e141d 2024-11-16T00:00:32,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8f87168d6e3e4545933717e6aebebba1 2024-11-16T00:00:32,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8102dd37cc284538bc82013a65f61047 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/8102dd37cc284538bc82013a65f61047 2024-11-16T00:00:32,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/bb8ae081cfbe400bab73f13162f057a3 2024-11-16T00:00:32,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c6c68ec69bc64fb388ba27e9102f8c62 2024-11-16T00:00:32,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/6923d157d3ac4eb28612c09eaf4cd4b1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/6923d157d3ac4eb28612c09eaf4cd4b1 2024-11-16T00:00:32,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/e6244c08b3dc42a99d72df3db0577540 2024-11-16T00:00:32,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c643b4ddd41244be946dde3922db7814 2024-11-16T00:00:32,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2128992b63541ba90fe65893f4a3014 2024-11-16T00:00:32,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/0cdccbf3fe664235ba5816243bf96396 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/0cdccbf3fe664235ba5816243bf96396 2024-11-16T00:00:32,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/c30cbd80824f4f7683fe66c1a0fe4d91 2024-11-16T00:00:32,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/282f818b87e243339e0e64ef758278aa 2024-11-16T00:00:32,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/3df271ec6be440919176f75d9ff50f19 2024-11-16T00:00:32,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f093484548774404bdb528c0d48bb17d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/32c93536b3db4e93b277db6c0c352ba3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f7dca9eb2ea74bbfaadf1e3c40a14131, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/84c46e8c04444d6aba1829a888fd9c07, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/b331d82501254396ad5dab7ceae1e913, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a3e378a97b654b568cf29e29219dfbfd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/753df2f526ad49858fec9860766000aa, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672] to archive 2024-11-16T00:00:32,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:00:32,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/61be4a9cf46e4caeac1a04c0f2480f6a 2024-11-16T00:00:32,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9d03411ebbdf40399d98c2dc82f5fefa 2024-11-16T00:00:32,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f093484548774404bdb528c0d48bb17d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f093484548774404bdb528c0d48bb17d 2024-11-16T00:00:32,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/11802990c3674cb58adf143fed2efda1 2024-11-16T00:00:32,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f32dc840124747a1baef0afe87d6612a 2024-11-16T00:00:32,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/86be293bfdf7425fb963078b3add49ef 2024-11-16T00:00:32,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/32c93536b3db4e93b277db6c0c352ba3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/32c93536b3db4e93b277db6c0c352ba3 2024-11-16T00:00:32,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/ab9899efe66444d8a56758c11ee5bac3 2024-11-16T00:00:32,941 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/9a1c0af77dc04d0cab925d7b04543c10 2024-11-16T00:00:32,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/beb9d72f0c054dc0a80b954318661202 2024-11-16T00:00:32,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f7dca9eb2ea74bbfaadf1e3c40a14131 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/f7dca9eb2ea74bbfaadf1e3c40a14131 2024-11-16T00:00:32,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/def78aeb33c34d9a8d1f899ab059ede0 2024-11-16T00:00:32,944 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/e0b3056ee9574ed2bdb0ef77f176e88e 2024-11-16T00:00:32,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/84c46e8c04444d6aba1829a888fd9c07 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/84c46e8c04444d6aba1829a888fd9c07 2024-11-16T00:00:32,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c81dc1cf913e4175a68ff2c4ef52c25e 2024-11-16T00:00:32,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/eb85a053ff18453b9539f7ff8fbc7ce3 2024-11-16T00:00:32,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/b331d82501254396ad5dab7ceae1e913 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/b331d82501254396ad5dab7ceae1e913 2024-11-16T00:00:32,948 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/af042469cd774013a4075a3fb0ba011e 2024-11-16T00:00:32,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d5d8a87114474338bf0bf2d90a959cce 2024-11-16T00:00:32,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a3e378a97b654b568cf29e29219dfbfd to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a3e378a97b654b568cf29e29219dfbfd 2024-11-16T00:00:32,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d6a50e73598e458b84d80147188ba423 2024-11-16T00:00:32,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/7659314d36ad404aa7a28832f35b5d61 2024-11-16T00:00:32,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/874d9e24f0b744de8a9b42baae66a5d4 2024-11-16T00:00:32,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/753df2f526ad49858fec9860766000aa to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/753df2f526ad49858fec9860766000aa 2024-11-16T00:00:32,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/53982595e1f64c5799fde4a6ae858a5e 2024-11-16T00:00:32,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/a22c7646765a48d7bb1670b3363b12b4 2024-11-16T00:00:32,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d1b1885321614139ab1232c5bb27d672 2024-11-16T00:00:33,147 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:33,150 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674276463ea274a79856955a8c226e534_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674276463ea274a79856955a8c226e534_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:33,151 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e48d477eb06846b392a87189fca4c3af, store: [table=TestAcidGuarantees family=A region=9d50d9faa764c624311a6377059225df] 2024-11-16T00:00:33,152 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e48d477eb06846b392a87189fca4c3af is 175, key is test_row_0/A:col10/1731715231070/Put/seqid=0 2024-11-16T00:00:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742294_1470 (size=22561) 2024-11-16T00:00:33,158 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e48d477eb06846b392a87189fca4c3af 2024-11-16T00:00:33,164 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/484ec434ac5f40b29e776628286550df is 50, key is test_row_0/B:col10/1731715231070/Put/seqid=0 2024-11-16T00:00:33,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742295_1471 (size=9857) 2024-11-16T00:00:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:33,571 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/484ec434ac5f40b29e776628286550df 2024-11-16T00:00:33,578 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5555d5bd382479b87ed50530e45cffc is 50, key is test_row_0/C:col10/1731715231070/Put/seqid=0 2024-11-16T00:00:33,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742296_1472 (size=9857) 2024-11-16T00:00:33,985 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5555d5bd382479b87ed50530e45cffc 2024-11-16T00:00:33,989 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/A/e48d477eb06846b392a87189fca4c3af as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e48d477eb06846b392a87189fca4c3af 2024-11-16T00:00:33,993 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e48d477eb06846b392a87189fca4c3af, entries=100, sequenceid=431, filesize=22.0 K 2024-11-16T00:00:33,993 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/B/484ec434ac5f40b29e776628286550df as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/484ec434ac5f40b29e776628286550df 2024-11-16T00:00:33,996 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/484ec434ac5f40b29e776628286550df, entries=100, sequenceid=431, filesize=9.6 K 2024-11-16T00:00:33,997 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/.tmp/C/c5555d5bd382479b87ed50530e45cffc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5555d5bd382479b87ed50530e45cffc 2024-11-16T00:00:34,000 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5555d5bd382479b87ed50530e45cffc, entries=100, sequenceid=431, filesize=9.6 K 2024-11-16T00:00:34,000 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 9d50d9faa764c624311a6377059225df in 1267ms, sequenceid=431, compaction requested=true 2024-11-16T00:00:34,011 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=4 2024-11-16T00:00:34,011 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. 2024-11-16T00:00:34,012 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 9d50d9faa764c624311a6377059225df: Waiting for close lock at 1731715231585Running coprocessor pre-close hooks at 1731715231585Disabling compacts and flushes for region at 1731715231585Disabling writes for close at 1731715232733 (+1148 ms)Obtaining lock to block concurrent updates at 1731715232733Preparing flush snapshotting stores in 9d50d9faa764c624311a6377059225df at 1731715232733Finished memstore snapshotting TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731715232733Flushing stores of TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df. at 1731715232733Flushing 9d50d9faa764c624311a6377059225df/A: creating writer at 1731715232733Flushing mob file 9d50d9faa764c624311a6377059225df/A: appending metadata at 1731715232737 (+4 ms)Flushing mob file 9d50d9faa764c624311a6377059225df/A: closing flushed file at 1731715232737Flushing 9d50d9faa764c624311a6377059225df/A: appending metadata at 1731715233151 (+414 ms)Flushing 9d50d9faa764c624311a6377059225df/A: closing flushed file at 1731715233151Flushing 9d50d9faa764c624311a6377059225df/B: creating writer at 1731715233161 (+10 ms)Flushing 9d50d9faa764c624311a6377059225df/B: appending metadata at 1731715233163 (+2 ms)Flushing 9d50d9faa764c624311a6377059225df/B: closing flushed file at 1731715233163Flushing 9d50d9faa764c624311a6377059225df/C: creating writer at 1731715233575 (+412 ms)Flushing 9d50d9faa764c624311a6377059225df/C: appending metadata at 1731715233577 (+2 ms)Flushing 9d50d9faa764c624311a6377059225df/C: closing flushed file at 1731715233577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c401d38: reopening flushed file at 1731715233989 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ad9b9bc: reopening flushed file at 1731715233993 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@394fbf31: reopening flushed file at 1731715233996 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 9d50d9faa764c624311a6377059225df in 1267ms, sequenceid=431, compaction requested=true at 1731715234000 (+4 ms)Writing region close event to WAL at 1731715234007 (+7 ms)Running coprocessor post-close hooks at 1731715234011 (+4 ms)Closed at 1731715234011 2024-11-16T00:00:34,014 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 9d50d9faa764c624311a6377059225df 2024-11-16T00:00:34,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9d50d9faa764c624311a6377059225df, regionState=CLOSED 2024-11-16T00:00:34,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:34,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T00:00:34,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d50d9faa764c624311a6377059225df, server=0801813e30ae,43013,1731715110098 in 2.5840 sec 2024-11-16T00:00:34,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-16T00:00:34,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d50d9faa764c624311a6377059225df, UNASSIGN in 2.5890 sec 2024-11-16T00:00:34,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T00:00:34,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5920 sec 2024-11-16T00:00:34,024 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715234024"}]},"ts":"1731715234024"} 2024-11-16T00:00:34,025 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T00:00:34,026 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T00:00:34,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6040 sec 2024-11-16T00:00:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T00:00:35,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:35,590 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-16T00:00:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,592 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T00:00:35,593 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits] 2024-11-16T00:00:35,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/0f8ed03684c7490d9f903618c16120fc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/0f8ed03684c7490d9f903618c16120fc 2024-11-16T00:00:35,600 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/7099aec9b7e44466b6f4c0c7d213f36b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/7099aec9b7e44466b6f4c0c7d213f36b 2024-11-16T00:00:35,600 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e48d477eb06846b392a87189fca4c3af to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e48d477eb06846b392a87189fca4c3af 2024-11-16T00:00:35,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e7f1982209e64590a3b4e985975daf3e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/A/e7f1982209e64590a3b4e985975daf3e 2024-11-16T00:00:35,603 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/484ec434ac5f40b29e776628286550df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/484ec434ac5f40b29e776628286550df 2024-11-16T00:00:35,604 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2dfe9753512436ab0f8323ef8c347e4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/a2dfe9753512436ab0f8323ef8c347e4 2024-11-16T00:00:35,605 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0d7f17564db48aba49ecc7796697a30 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/b0d7f17564db48aba49ecc7796697a30 2024-11-16T00:00:35,606 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74e0d7357604fcea2b10c665ac55555 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/B/d74e0d7357604fcea2b10c665ac55555 2024-11-16T00:00:35,608 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/3457e66d622443399e8adf0ffaf1abf0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/3457e66d622443399e8adf0ffaf1abf0 2024-11-16T00:00:35,609 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5555d5bd382479b87ed50530e45cffc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5555d5bd382479b87ed50530e45cffc 2024-11-16T00:00:35,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5f5e6da435b408ebc1df822e9a8d1b7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/c5f5e6da435b408ebc1df822e9a8d1b7 2024-11-16T00:00:35,611 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d808129a8f65443c96ee54216eb19324 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/C/d808129a8f65443c96ee54216eb19324 2024-11-16T00:00:35,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits/434.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df/recovered.edits/434.seqid 2024-11-16T00:00:35,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,614 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T00:00:35,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T00:00:35,615 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T00:00:35,617 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c38944a1496456889efea7bedfa6b22_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c38944a1496456889efea7bedfa6b22_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116267fc80dc17b4ca09345b5545918d42c_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116267fc80dc17b4ca09345b5545918d42c_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628a7f1ed92f542c6a056d374acb6d339_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111628a7f1ed92f542c6a056d374acb6d339_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ecd961f889c40c490bd5689a94ed51b_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162ecd961f889c40c490bd5689a94ed51b_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,621 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111635070a5e544742b2a31e9e443b5d9c17_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111635070a5e544742b2a31e9e443b5d9c17_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,622 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164af86b9e871440bbbe1db843d36a65b3_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164af86b9e871440bbbe1db843d36a65b3_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,623 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669fc30b45cb54e2dbff5270b1391d246_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111669fc30b45cb54e2dbff5270b1391d246_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f89d4f14f45409d8ce5df714c8dd39a_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f89d4f14f45409d8ce5df714c8dd39a_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674276463ea274a79856955a8c226e534_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674276463ea274a79856955a8c226e534_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674283df748a74f0ea1cd52f580c29a5d_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674283df748a74f0ea1cd52f580c29a5d_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111683b6492b9d52481fb4d9fca456c25711_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111683b6492b9d52481fb4d9fca456c25711_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,628 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111684c2ee387bff4254aea343dfad7da54a_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111684c2ee387bff4254aea343dfad7da54a_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116adf20bc60fcc4bc88b89b7e3acee7c5e_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116adf20bc60fcc4bc88b89b7e3acee7c5e_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1a79081f2414ce682d289e269f9a5c8_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1a79081f2414ce682d289e269f9a5c8_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,631 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b5be480bc40a4033a6810d0022299f5e_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b5be480bc40a4033a6810d0022299f5e_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bcbf85cde4be47258cbc6fcc630788fc_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bcbf85cde4be47258cbc6fcc630788fc_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd39253131eb4457b8bf369343c61671_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd39253131eb4457b8bf369343c61671_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c159287367b44afea46d9f5d04f53f4d_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c159287367b44afea46d9f5d04f53f4d_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5dfb1a88e2d4952b3ebb44e8dceac59_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c5dfb1a88e2d4952b3ebb44e8dceac59_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0842cf2c4b443bcbcd813e4ae425971_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0842cf2c4b443bcbcd813e4ae425971_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,637 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0b975edfbd84f1b831746bf8735f016_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d0b975edfbd84f1b831746bf8735f016_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2b67fbbf46b4fd2905d1ec36aa896ea_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e2b67fbbf46b4fd2905d1ec36aa896ea_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,639 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f375c878f4e84c1db57fa4c021a34853_9d50d9faa764c624311a6377059225df to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f375c878f4e84c1db57fa4c021a34853_9d50d9faa764c624311a6377059225df 2024-11-16T00:00:35,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T00:00:35,643 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,645 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T00:00:35,647 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T00:00:35,647 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,647 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T00:00:35,648 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715235647"}]},"ts":"9223372036854775807"} 2024-11-16T00:00:35,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T00:00:35,649 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9d50d9faa764c624311a6377059225df, NAME => 'TestAcidGuarantees,,1731715207302.9d50d9faa764c624311a6377059225df.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T00:00:35,649 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T00:00:35,650 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715235649"}]},"ts":"9223372036854775807"} 2024-11-16T00:00:35,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T00:00:35,652 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-16T00:00:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T00:00:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T00:00:35,700 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:35,713 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 235) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1703801164_22 at /127.0.0.1:38008 [Waiting for operation #1192] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1703801164_22 at /127.0.0.1:35936 [Waiting for operation #1391] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/0801813e30ae:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1417717973_22 at /127.0.0.1:38010 [Waiting for operation #1188] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1417717973_22 at /127.0.0.1:38004 [Waiting for operation #1216] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=651 (was 694), ProcessCount=11 (was 11), AvailableMemoryMB=5003 (was 4603) - AvailableMemoryMB LEAK? - 2024-11-16T00:00:35,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=651, ProcessCount=11, AvailableMemoryMB=5003 2024-11-16T00:00:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T00:00:35,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T00:00:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:35,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T00:00:35,728 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:35,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-16T00:00:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:35,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T00:00:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742297_1473 (size=959) 2024-11-16T00:00:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:36,136 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-16T00:00:36,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742298_1474 (size=53) 2024-11-16T00:00:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e6d31893a761fb01419b5d3147eeec1a, disabling compactions & flushes 2024-11-16T00:00:36,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. after waiting 0 ms 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,547 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6d31893a761fb01419b5d3147eeec1a: Waiting for close lock at 1731715236547Disabling compacts and flushes for region at 1731715236547Disabling writes for close at 1731715236547Writing region close event to WAL at 1731715236547Closed at 1731715236547 2024-11-16T00:00:36,549 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T00:00:36,549 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715236549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715236549"}]},"ts":"1731715236549"} 2024-11-16T00:00:36,551 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T00:00:36,551 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T00:00:36,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715236551"}]},"ts":"1731715236551"} 2024-11-16T00:00:36,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T00:00:36,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, ASSIGN}] 2024-11-16T00:00:36,554 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, ASSIGN 2024-11-16T00:00:36,555 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-16T00:00:36,706 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e6d31893a761fb01419b5d3147eeec1a, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:36,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, ASSIGN because future has completed 2024-11-16T00:00:36,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:36,862 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,862 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T00:00:36,863 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,863 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:00:36,863 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,863 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,864 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,866 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d31893a761fb01419b5d3147eeec1a columnFamilyName A 2024-11-16T00:00:36,866 DEBUG [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:36,866 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(327): Store=e6d31893a761fb01419b5d3147eeec1a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:36,867 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,868 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d31893a761fb01419b5d3147eeec1a columnFamilyName B 2024-11-16T00:00:36,868 DEBUG [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:36,868 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(327): Store=e6d31893a761fb01419b5d3147eeec1a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:36,868 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,869 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d31893a761fb01419b5d3147eeec1a columnFamilyName C 2024-11-16T00:00:36,869 DEBUG [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:00:36,870 INFO [StoreOpener-e6d31893a761fb01419b5d3147eeec1a-1 {}] regionserver.HStore(327): Store=e6d31893a761fb01419b5d3147eeec1a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:00:36,870 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:36,870 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,871 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,871 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,872 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,872 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T00:00:36,873 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,874 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T00:00:36,875 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened e6d31893a761fb01419b5d3147eeec1a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61752275, jitterRate=-0.07981939613819122}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T00:00:36,875 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:36,875 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for e6d31893a761fb01419b5d3147eeec1a: Running coprocessor pre-open hook at 1731715236863Writing region info on filesystem at 1731715236863Initializing all the Stores at 1731715236864 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715236864Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715236864Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715236864Cleaning up temporary data from old regions at 1731715236872 (+8 ms)Running coprocessor post-open hooks at 1731715236875 (+3 ms)Region opened successfully at 1731715236875 2024-11-16T00:00:36,876 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., pid=118, masterSystemTime=1731715236859 2024-11-16T00:00:36,878 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,878 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:36,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e6d31893a761fb01419b5d3147eeec1a, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:36,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:00:36,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T00:00:36,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 in 173 msec 2024-11-16T00:00:36,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-16T00:00:36,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, ASSIGN in 329 msec 2024-11-16T00:00:36,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T00:00:36,885 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715236885"}]},"ts":"1731715236885"} 2024-11-16T00:00:36,886 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T00:00:36,887 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T00:00:36,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-16T00:00:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-16T00:00:37,880 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:37,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e9a9b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fed5c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,884 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59504c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,885 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,887 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T00:00:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-16T00:00:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a291c6c 2024-11-16T00:00:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T00:00:37,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T00:00:37,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ded6cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25882bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165a2ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27bfce74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@865e19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,894 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564d4707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a52e343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6e796a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e16009a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335cae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,898 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d5626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7318ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76919fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cae88aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7f3a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf63333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,903 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa7beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,904 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f41cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fd006e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1f70ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7013bf69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fddce7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d22571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c40767f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cec2e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b7094e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,912 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20381111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c25862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:00:37,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:00:37,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7981967f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:00:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:37,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:00:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6229cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:00:37,916 DEBUG [Thread-2115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,916 DEBUG [Thread-2121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,916 DEBUG [Thread-2117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,916 DEBUG [Thread-2123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:37,917 DEBUG [Thread-2126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,917 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,917 DEBUG [Thread-2128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,917 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,918 DEBUG [Thread-2130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,918 DEBUG [Thread-2119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:00:37,919 DEBUG [Thread-2130 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,920 DEBUG [Thread-2130 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,920 DEBUG [Thread-2126 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:00:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:37,920 DEBUG [Thread-2126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:00:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,923 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:37,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:37,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:00:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2] 2024-11-16T00:00:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:37,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T00:00:37,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/f56b3b11c89d461882cee6a08ab8abb2 is 50, key is test_row_0/A:col10/1731715237932/Put/seqid=0 2024-11-16T00:00:37,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742299_1475 (size=14341) 2024-11-16T00:00:37,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/f56b3b11c89d461882cee6a08ab8abb2 2024-11-16T00:00:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715297973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:37,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715297981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715297986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715297988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715297989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8d0c90841c9e42ca9b924e31a604e63a is 50, key is test_row_0/B:col10/1731715237932/Put/seqid=0 2024-11-16T00:00:38,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742300_1476 (size=12001) 2024-11-16T00:00:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:38,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T00:00:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715298100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715298100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715298109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715298109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715298109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T00:00:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715298309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715298309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715298319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715298330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715298330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T00:00:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:38,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8d0c90841c9e42ca9b924e31a604e63a 2024-11-16T00:00:38,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2f8a04728d7848598801da4b7063c428 is 50, key is test_row_0/C:col10/1731715237932/Put/seqid=0 2024-11-16T00:00:38,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742301_1477 (size=12001) 2024-11-16T00:00:38,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2f8a04728d7848598801da4b7063c428 2024-11-16T00:00:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/f56b3b11c89d461882cee6a08ab8abb2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2 2024-11-16T00:00:38,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2, entries=200, sequenceid=15, filesize=14.0 K 2024-11-16T00:00:38,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8d0c90841c9e42ca9b924e31a604e63a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a 2024-11-16T00:00:38,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T00:00:38,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2f8a04728d7848598801da4b7063c428 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428 2024-11-16T00:00:38,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T00:00:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e6d31893a761fb01419b5d3147eeec1a in 557ms, sequenceid=15, compaction requested=false 2024-11-16T00:00:38,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:38,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T00:00:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:38,546 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T00:00:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0d8087bca775407aaae6536343474bf8 is 50, key is test_row_0/A:col10/1731715237986/Put/seqid=0 2024-11-16T00:00:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742302_1478 (size=12001) 2024-11-16T00:00:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0d8087bca775407aaae6536343474bf8 2024-11-16T00:00:38,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f4757903f844ca6aa0e5b38e973b770 is 50, key is test_row_0/B:col10/1731715237986/Put/seqid=0 2024-11-16T00:00:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742303_1479 (size=12001) 2024-11-16T00:00:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715298659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715298663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715298666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715298669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715298678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715298781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715298790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715298793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715298799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715298799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715299000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715299000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715299000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,008 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f4757903f844ca6aa0e5b38e973b770 2024-11-16T00:00:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/30ae12151ce94448a8a4cf67b06c9bfe is 50, key is test_row_0/C:col10/1731715237986/Put/seqid=0 2024-11-16T00:00:39,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715299020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715299020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742304_1480 (size=12001) 2024-11-16T00:00:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715299310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715299310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715299311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715299341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715299341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,437 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/30ae12151ce94448a8a4cf67b06c9bfe 2024-11-16T00:00:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0d8087bca775407aaae6536343474bf8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8 2024-11-16T00:00:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T00:00:39,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f4757903f844ca6aa0e5b38e973b770 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770 2024-11-16T00:00:39,459 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T00:00:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/30ae12151ce94448a8a4cf67b06c9bfe as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe 2024-11-16T00:00:39,463 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T00:00:39,464 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e6d31893a761fb01419b5d3147eeec1a in 918ms, sequenceid=37, compaction requested=false 2024-11-16T00:00:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T00:00:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T00:00:39,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T00:00:39,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-16T00:00:39,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5520 sec 2024-11-16T00:00:39,538 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T00:00:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:39,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:39,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T00:00:39,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/eb7e2a4f7f5145089be06348a6e848cc is 50, key is test_row_0/A:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:39,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742305_1481 (size=12001) 2024-11-16T00:00:39,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/eb7e2a4f7f5145089be06348a6e848cc 2024-11-16T00:00:39,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/54099feee58542518971a004baa2906d is 50, key is test_row_0/B:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:39,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715299897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715299900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715299902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742306_1482 (size=12001) 2024-11-16T00:00:39,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/54099feee58542518971a004baa2906d 2024-11-16T00:00:39,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715299908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715299914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:39,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f499777a377c486e960114f779af6e06 is 50, key is test_row_0/C:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:39,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742307_1483 (size=12001) 2024-11-16T00:00:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715300020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715300020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715300021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715300046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715300049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T00:00:40,070 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:40,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T00:00:40,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:40,074 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:40,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:40,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T00:00:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715300242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715300242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715300242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715300271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715300271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T00:00:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:40,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:40,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f499777a377c486e960114f779af6e06 2024-11-16T00:00:40,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/eb7e2a4f7f5145089be06348a6e848cc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc 2024-11-16T00:00:40,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T00:00:40,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/54099feee58542518971a004baa2906d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d 2024-11-16T00:00:40,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T00:00:40,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f499777a377c486e960114f779af6e06 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06 2024-11-16T00:00:40,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T00:00:40,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e6d31893a761fb01419b5d3147eeec1a in 589ms, sequenceid=52, compaction requested=true 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:40,424 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:40,424 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:40,425 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:40,425 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:40,425 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,425 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.2 K 2024-11-16T00:00:40,425 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:40,425 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:40,425 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,425 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=37.4 K 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56b3b11c89d461882cee6a08ab8abb2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715237932 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0c90841c9e42ca9b924e31a604e63a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715237932 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4757903f844ca6aa0e5b38e973b770, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715237983 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d8087bca775407aaae6536343474bf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715237983 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 54099feee58542518971a004baa2906d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:40,426 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7e2a4f7f5145089be06348a6e848cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:40,436 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:40,437 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/eed1c70a911949cd8e13823de612b415 is 50, key is test_row_0/B:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:40,444 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:40,445 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0ab6a1dc44b64347a65270653378be51 is 50, key is test_row_0/A:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:40,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742308_1484 (size=12104) 2024-11-16T00:00:40,456 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/eed1c70a911949cd8e13823de612b415 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/eed1c70a911949cd8e13823de612b415 2024-11-16T00:00:40,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742309_1485 (size=12104) 2024-11-16T00:00:40,470 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into eed1c70a911949cd8e13823de612b415(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:40,470 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:40,470 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715240424; duration=0sec 2024-11-16T00:00:40,470 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:40,470 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:40,470 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:40,472 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:40,472 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:40,472 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,472 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.2 K 2024-11-16T00:00:40,473 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f8a04728d7848598801da4b7063c428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731715237932 2024-11-16T00:00:40,473 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0ab6a1dc44b64347a65270653378be51 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0ab6a1dc44b64347a65270653378be51 2024-11-16T00:00:40,473 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ae12151ce94448a8a4cf67b06c9bfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731715237983 2024-11-16T00:00:40,474 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f499777a377c486e960114f779af6e06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:40,481 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:40,482 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/5af5b221d65b4156915c815bc37a8ebb is 50, key is test_row_0/C:col10/1731715239833/Put/seqid=0 2024-11-16T00:00:40,484 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 0ab6a1dc44b64347a65270653378be51(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:40,484 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:40,484 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715240424; duration=0sec 2024-11-16T00:00:40,484 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:40,484 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742310_1486 (size=12104) 2024-11-16T00:00:40,518 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/5af5b221d65b4156915c815bc37a8ebb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/5af5b221d65b4156915c815bc37a8ebb 2024-11-16T00:00:40,527 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 5af5b221d65b4156915c815bc37a8ebb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:40,527 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:40,527 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715240424; duration=0sec 2024-11-16T00:00:40,527 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:40,527 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:40,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T00:00:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:40,545 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T00:00:40,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ffecf38b629e4b008b918d84bedcb245 is 50, key is test_row_0/A:col10/1731715239895/Put/seqid=0 2024-11-16T00:00:40,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:40,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742311_1487 (size=12001) 2024-11-16T00:00:40,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715300590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715300593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715300595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715300598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715300599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:40,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715300710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715300710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715300719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715300720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715300930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715300930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715300930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715300940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:40,978 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ffecf38b629e4b008b918d84bedcb245 2024-11-16T00:00:41,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc1d0dc2322d4ab0a168bc072850403b is 50, key is test_row_0/B:col10/1731715239895/Put/seqid=0 2024-11-16T00:00:41,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742312_1488 (size=12001) 2024-11-16T00:00:41,020 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc1d0dc2322d4ab0a168bc072850403b 2024-11-16T00:00:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d1c7924189f64945bf6c0e96f3279384 is 50, key is test_row_0/C:col10/1731715239895/Put/seqid=0 2024-11-16T00:00:41,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742313_1489 (size=12001) 2024-11-16T00:00:41,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715301110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715301240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715301241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715301242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715301260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,488 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d1c7924189f64945bf6c0e96f3279384 2024-11-16T00:00:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ffecf38b629e4b008b918d84bedcb245 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245 2024-11-16T00:00:41,496 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T00:00:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc1d0dc2322d4ab0a168bc072850403b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b 2024-11-16T00:00:41,502 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T00:00:41,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d1c7924189f64945bf6c0e96f3279384 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384 2024-11-16T00:00:41,506 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T00:00:41,507 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e6d31893a761fb01419b5d3147eeec1a in 961ms, sequenceid=76, compaction requested=false 2024-11-16T00:00:41,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:41,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:41,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T00:00:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T00:00:41,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T00:00:41,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-16T00:00:41,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4410 sec 2024-11-16T00:00:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:41,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T00:00:41,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/9b26abc40c354c8bb725a8fd329b0f3f is 50, key is test_row_0/A:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:41,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742314_1490 (size=14341) 2024-11-16T00:00:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715301832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715301834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715301835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715301844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715301960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715301960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715301960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715301960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,143 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_2' in e6d31893a761fb01419b5d3147eeec1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T00:00:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715302169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715302169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715302173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715302184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/9b26abc40c354c8bb725a8fd329b0f3f 2024-11-16T00:00:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f76e4afb7734b8c96a83d37a3e861fb is 50, key is test_row_0/B:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:42,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742315_1491 (size=12001) 2024-11-16T00:00:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T00:00:42,230 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:42,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T00:00:42,232 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:42,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:42,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:42,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T00:00:42,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:42,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715302489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715302489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715302500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715302510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:42,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T00:00:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:42,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f76e4afb7734b8c96a83d37a3e861fb 2024-11-16T00:00:42,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0c632d1b47ae4e06a86a01ec16097f5f is 50, key is test_row_0/C:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:42,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742316_1492 (size=12001) 2024-11-16T00:00:42,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0c632d1b47ae4e06a86a01ec16097f5f 2024-11-16T00:00:42,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/9b26abc40c354c8bb725a8fd329b0f3f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f 2024-11-16T00:00:42,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f, entries=200, sequenceid=92, filesize=14.0 K 2024-11-16T00:00:42,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8f76e4afb7734b8c96a83d37a3e861fb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb 2024-11-16T00:00:42,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T00:00:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:42,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T00:00:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0c632d1b47ae4e06a86a01ec16097f5f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f 2024-11-16T00:00:42,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T00:00:42,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e6d31893a761fb01419b5d3147eeec1a in 949ms, sequenceid=92, compaction requested=true 2024-11-16T00:00:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:42,716 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:42,717 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:42,717 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:42,717 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,717 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0ab6a1dc44b64347a65270653378be51, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=37.5 K 2024-11-16T00:00:42,718 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab6a1dc44b64347a65270653378be51, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:42,718 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffecf38b629e4b008b918d84bedcb245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715239878 2024-11-16T00:00:42,719 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b26abc40c354c8bb725a8fd329b0f3f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:42,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:42,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:42,728 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:42,728 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:42,728 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,728 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/eed1c70a911949cd8e13823de612b415, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.3 K 2024-11-16T00:00:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:42,729 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting eed1c70a911949cd8e13823de612b415, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:42,729 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1d0dc2322d4ab0a168bc072850403b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715239878 2024-11-16T00:00:42,730 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f76e4afb7734b8c96a83d37a3e861fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:42,735 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:42,735 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/6930ff874eab499898721b780299e6a7 is 50, key is test_row_0/A:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:42,742 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:42,743 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/2dd818dc305b4e52a5c14fe8380df026 is 50, key is test_row_0/B:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:42,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742317_1493 (size=12207) 2024-11-16T00:00:42,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742318_1494 (size=12207) 2024-11-16T00:00:42,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T00:00:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:42,854 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:00:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a7d23b79a90c40f8a023f6f6b063667c is 50, key is test_row_0/A:col10/1731715241838/Put/seqid=0 2024-11-16T00:00:42,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742319_1495 (size=12001) 2024-11-16T00:00:43,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:43,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715303070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715303073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715303074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715303085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,184 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/6930ff874eab499898721b780299e6a7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6930ff874eab499898721b780299e6a7 2024-11-16T00:00:43,185 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/2dd818dc305b4e52a5c14fe8380df026 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/2dd818dc305b4e52a5c14fe8380df026 2024-11-16T00:00:43,200 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into 2dd818dc305b4e52a5c14fe8380df026(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:43,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:43,200 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715242727; duration=0sec 2024-11-16T00:00:43,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:43,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:43,200 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:43,201 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 6930ff874eab499898721b780299e6a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:43,201 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:43,201 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715242716; duration=0sec 2024-11-16T00:00:43,201 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:43,201 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:43,201 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:43,202 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:43,202 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:43,202 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/5af5b221d65b4156915c815bc37a8ebb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.3 K 2024-11-16T00:00:43,202 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af5b221d65b4156915c815bc37a8ebb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731715238661 2024-11-16T00:00:43,202 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c7924189f64945bf6c0e96f3279384, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715239878 2024-11-16T00:00:43,203 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c632d1b47ae4e06a86a01ec16097f5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715303190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715303190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715303200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715303209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,231 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:43,231 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/a3db7d7bee1f47258f2b2730ad402d2d is 50, key is test_row_0/C:col10/1731715240567/Put/seqid=0 2024-11-16T00:00:43,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742320_1496 (size=12207) 2024-11-16T00:00:43,266 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/a3db7d7bee1f47258f2b2730ad402d2d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a3db7d7bee1f47258f2b2730ad402d2d 2024-11-16T00:00:43,271 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into a3db7d7bee1f47258f2b2730ad402d2d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:43,271 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:43,271 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715242729; duration=0sec 2024-11-16T00:00:43,271 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:43,271 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:43,288 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a7d23b79a90c40f8a023f6f6b063667c 2024-11-16T00:00:43,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d97d3e6e79af4243b51d571d6a1cbb9a is 50, key is test_row_0/B:col10/1731715241838/Put/seqid=0 2024-11-16T00:00:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742321_1497 (size=12001) 2024-11-16T00:00:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715303419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715303429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715303430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715303430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,712 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d97d3e6e79af4243b51d571d6a1cbb9a 2024-11-16T00:00:43,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/037d4817a43a4d3e903350dca4eb5370 is 50, key is test_row_0/C:col10/1731715241838/Put/seqid=0 2024-11-16T00:00:43,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742322_1498 (size=12001) 2024-11-16T00:00:43,739 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/037d4817a43a4d3e903350dca4eb5370 2024-11-16T00:00:43,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a7d23b79a90c40f8a023f6f6b063667c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c 2024-11-16T00:00:43,748 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T00:00:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d97d3e6e79af4243b51d571d6a1cbb9a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a 2024-11-16T00:00:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715303740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715303740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715303740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715303740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:43,761 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T00:00:43,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/037d4817a43a4d3e903350dca4eb5370 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370 2024-11-16T00:00:43,770 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T00:00:43,771 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e6d31893a761fb01419b5d3147eeec1a in 916ms, sequenceid=112, compaction requested=false 2024-11-16T00:00:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T00:00:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T00:00:43,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T00:00:43,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-11-16T00:00:43,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5450 sec 2024-11-16T00:00:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:00:44,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/537e3324931b41f8b941e5aa1cfbd472 is 50, key is test_row_0/A:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742323_1499 (size=14391) 2024-11-16T00:00:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715304288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715304289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715304290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715304290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715304304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T00:00:44,382 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:44,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T00:00:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:44,384 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:44,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:44,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715304410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715304411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715304419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715304420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715304431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:44,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T00:00:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:44,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/537e3324931b41f8b941e5aa1cfbd472 2024-11-16T00:00:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715304630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715304632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc2e502345714b5daa5a09fe211b67b5 is 50, key is test_row_0/B:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:44,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742324_1500 (size=12051) 2024-11-16T00:00:44,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc2e502345714b5daa5a09fe211b67b5 2024-11-16T00:00:44,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715304640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715304642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/9fe2aabd0b9542f491ed9775c3218e29 is 50, key is test_row_0/C:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715304650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742325_1501 (size=12051) 2024-11-16T00:00:44,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T00:00:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:44,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T00:00:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:44,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:44,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715304950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715304950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715304960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715304970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715304979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T00:00:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:45,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/9fe2aabd0b9542f491ed9775c3218e29 2024-11-16T00:00:45,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/537e3324931b41f8b941e5aa1cfbd472 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472 2024-11-16T00:00:45,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472, entries=200, sequenceid=132, filesize=14.1 K 2024-11-16T00:00:45,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/fc2e502345714b5daa5a09fe211b67b5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5 2024-11-16T00:00:45,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T00:00:45,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/9fe2aabd0b9542f491ed9775c3218e29 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29 2024-11-16T00:00:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T00:00:45,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e6d31893a761fb01419b5d3147eeec1a in 913ms, sequenceid=132, compaction requested=true 2024-11-16T00:00:45,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:45,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:45,091 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:45,091 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:45,091 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:45,092 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:45,092 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,092 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6930ff874eab499898721b780299e6a7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=37.7 K 2024-11-16T00:00:45,092 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:45,092 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6930ff874eab499898721b780299e6a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:45,092 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:45,092 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,092 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/2dd818dc305b4e52a5c14fe8380df026, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.4 K 2024-11-16T00:00:45,093 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d23b79a90c40f8a023f6f6b063667c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731715241832 2024-11-16T00:00:45,093 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd818dc305b4e52a5c14fe8380df026, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:45,093 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d97d3e6e79af4243b51d571d6a1cbb9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731715241832 2024-11-16T00:00:45,093 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537e3324931b41f8b941e5aa1cfbd472, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:45,094 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2e502345714b5daa5a09fe211b67b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:45,103 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:45,104 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/c7dfa7814a3e4939832fb71a5c05d84f is 50, key is test_row_0/A:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:45,109 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:45,109 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a0d89f04f4ac469db3491af07aa9e223 is 50, key is test_row_0/B:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:45,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742326_1502 (size=12359) 2024-11-16T00:00:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742327_1503 (size=12359) 2024-11-16T00:00:45,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T00:00:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,154 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:00:45,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4e9e3f46d23441198cb65707bbefc26b is 50, key is test_row_0/A:col10/1731715244285/Put/seqid=0 2024-11-16T00:00:45,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742328_1504 (size=12151) 2024-11-16T00:00:45,171 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4e9e3f46d23441198cb65707bbefc26b 2024-11-16T00:00:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/cb6bdcb996c6483a94c0988ff3fffb6b is 50, key is test_row_0/B:col10/1731715244285/Put/seqid=0 2024-11-16T00:00:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742329_1505 (size=12151) 2024-11-16T00:00:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:45,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:45,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715305518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,539 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/c7dfa7814a3e4939832fb71a5c05d84f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/c7dfa7814a3e4939832fb71a5c05d84f 2024-11-16T00:00:45,541 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a0d89f04f4ac469db3491af07aa9e223 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a0d89f04f4ac469db3491af07aa9e223 2024-11-16T00:00:45,546 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into c7dfa7814a3e4939832fb71a5c05d84f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:45,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:45,546 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715245090; duration=0sec 2024-11-16T00:00:45,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:45,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:45,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:45,548 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into a0d89f04f4ac469db3491af07aa9e223(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:45,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:45,548 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715245091; duration=0sec 2024-11-16T00:00:45,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:45,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:45,548 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:45,549 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:45,549 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,549 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a3db7d7bee1f47258f2b2730ad402d2d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.4 K 2024-11-16T00:00:45,549 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3db7d7bee1f47258f2b2730ad402d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731715240567 2024-11-16T00:00:45,550 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 037d4817a43a4d3e903350dca4eb5370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731715241832 2024-11-16T00:00:45,550 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe2aabd0b9542f491ed9775c3218e29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715305534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715305534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715305534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715305537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,571 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:45,572 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/bb7d1f0cb261469dbbed773ce3f8df07 is 50, key is test_row_0/C:col10/1731715243083/Put/seqid=0 2024-11-16T00:00:45,590 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/cb6bdcb996c6483a94c0988ff3fffb6b 2024-11-16T00:00:45,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742330_1506 (size=12359) 2024-11-16T00:00:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/06fee4d1edb341b69119df72f717bf57 is 50, key is test_row_0/C:col10/1731715244285/Put/seqid=0 2024-11-16T00:00:45,608 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/bb7d1f0cb261469dbbed773ce3f8df07 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bb7d1f0cb261469dbbed773ce3f8df07 2024-11-16T00:00:45,625 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into bb7d1f0cb261469dbbed773ce3f8df07(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:45,625 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:45,625 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715245091; duration=0sec 2024-11-16T00:00:45,626 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:45,626 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:45,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742331_1507 (size=12151) 2024-11-16T00:00:45,639 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/06fee4d1edb341b69119df72f717bf57 2024-11-16T00:00:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4e9e3f46d23441198cb65707bbefc26b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b 2024-11-16T00:00:45,649 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T00:00:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/cb6bdcb996c6483a94c0988ff3fffb6b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b 2024-11-16T00:00:45,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715305640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,655 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T00:00:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/06fee4d1edb341b69119df72f717bf57 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57 2024-11-16T00:00:45,664 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T00:00:45,665 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e6d31893a761fb01419b5d3147eeec1a in 511ms, sequenceid=151, compaction requested=false 2024-11-16T00:00:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T00:00:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T00:00:45,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T00:00:45,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2820 sec 2024-11-16T00:00:45,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.2870 sec 2024-11-16T00:00:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:45,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:45,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:00:45,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a65a5f29988e4f9cb8039d831f8e211e is 50, key is test_row_0/A:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742332_1508 (size=14541) 2024-11-16T00:00:45,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a65a5f29988e4f9cb8039d831f8e211e 2024-11-16T00:00:45,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/28a59f61633c469c803c61918c9f272e is 50, key is test_row_0/B:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715305718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715305720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742333_1509 (size=12151) 2024-11-16T00:00:45,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715305736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715305738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/28a59f61633c469c803c61918c9f272e 2024-11-16T00:00:45,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/c83a4a13510a4726b8ef5faad1274624 is 50, key is test_row_0/C:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742334_1510 (size=12151) 2024-11-16T00:00:45,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715305840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715305840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715305860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715305860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715305860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715306060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715306060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715306080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715306080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715306180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/c83a4a13510a4726b8ef5faad1274624 2024-11-16T00:00:46,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/a65a5f29988e4f9cb8039d831f8e211e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e 2024-11-16T00:00:46,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e, entries=200, sequenceid=172, filesize=14.2 K 2024-11-16T00:00:46,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/28a59f61633c469c803c61918c9f272e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e 2024-11-16T00:00:46,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T00:00:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/c83a4a13510a4726b8ef5faad1274624 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624 2024-11-16T00:00:46,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T00:00:46,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 555ms, sequenceid=172, compaction requested=true 2024-11-16T00:00:46,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:46,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:46,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:46,228 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:46,228 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:46,229 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,229 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,229 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a0d89f04f4ac469db3491af07aa9e223, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.8 K 2024-11-16T00:00:46,229 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/c7dfa7814a3e4939832fb71a5c05d84f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=38.1 K 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7dfa7814a3e4939832fb71a5c05d84f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:46,229 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d89f04f4ac469db3491af07aa9e223, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:46,230 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9e3f46d23441198cb65707bbefc26b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731715244285 2024-11-16T00:00:46,230 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6bdcb996c6483a94c0988ff3fffb6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731715244285 2024-11-16T00:00:46,230 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65a5f29988e4f9cb8039d831f8e211e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:46,230 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 28a59f61633c469c803c61918c9f272e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:46,239 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:46,240 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/bd8b1bf12906429dbd13757042e99071 is 50, key is test_row_0/A:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:46,243 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:46,243 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a552d9be4b9140f98b8d9e832dca5ee9 is 50, key is test_row_0/B:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:46,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742335_1511 (size=12561) 2024-11-16T00:00:46,258 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/bd8b1bf12906429dbd13757042e99071 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/bd8b1bf12906429dbd13757042e99071 2024-11-16T00:00:46,264 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into bd8b1bf12906429dbd13757042e99071(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:46,264 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:46,265 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715246227; duration=0sec 2024-11-16T00:00:46,265 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:46,265 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:46,265 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:46,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:46,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:46,266 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,266 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bb7d1f0cb261469dbbed773ce3f8df07, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=35.8 K 2024-11-16T00:00:46,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7d1f0cb261469dbbed773ce3f8df07, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715243051 2024-11-16T00:00:46,266 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06fee4d1edb341b69119df72f717bf57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731715244285 2024-11-16T00:00:46,267 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83a4a13510a4726b8ef5faad1274624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:46,281 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:46,282 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/adba1bc4ff9a47a4839929792e5bc60a is 50, key is test_row_0/C:col10/1731715245534/Put/seqid=0 2024-11-16T00:00:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742336_1512 (size=12561) 2024-11-16T00:00:46,298 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a552d9be4b9140f98b8d9e832dca5ee9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a552d9be4b9140f98b8d9e832dca5ee9 2024-11-16T00:00:46,313 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into a552d9be4b9140f98b8d9e832dca5ee9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:46,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:46,313 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715246227; duration=0sec 2024-11-16T00:00:46,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:46,313 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:46,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742337_1513 (size=12561) 2024-11-16T00:00:46,334 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/adba1bc4ff9a47a4839929792e5bc60a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/adba1bc4ff9a47a4839929792e5bc60a 2024-11-16T00:00:46,340 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into adba1bc4ff9a47a4839929792e5bc60a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:46,340 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:46,342 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715246228; duration=0sec 2024-11-16T00:00:46,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:46,342 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:46,394 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:46,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:00:46,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/73df2f06cc2c4e0882093135e99c3eba is 50, key is test_row_0/A:col10/1731715246389/Put/seqid=0 2024-11-16T00:00:46,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742338_1514 (size=14541) 2024-11-16T00:00:46,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/73df2f06cc2c4e0882093135e99c3eba 2024-11-16T00:00:46,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/702d87d41b7249c093619ade5fa97ddb is 50, key is test_row_0/B:col10/1731715246389/Put/seqid=0 2024-11-16T00:00:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715306435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715306439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742339_1515 (size=12151) 2024-11-16T00:00:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715306447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715306449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/702d87d41b7249c093619ade5fa97ddb 2024-11-16T00:00:46,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/7649f3898704409695fb587cee35eb75 is 50, key is test_row_0/C:col10/1731715246389/Put/seqid=0 2024-11-16T00:00:46,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742340_1516 (size=12151) 2024-11-16T00:00:46,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/7649f3898704409695fb587cee35eb75 2024-11-16T00:00:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/73df2f06cc2c4e0882093135e99c3eba as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba 2024-11-16T00:00:46,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba, entries=200, sequenceid=194, filesize=14.2 K 2024-11-16T00:00:46,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/702d87d41b7249c093619ade5fa97ddb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb 2024-11-16T00:00:46,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T00:00:46,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/7649f3898704409695fb587cee35eb75 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75 2024-11-16T00:00:46,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T00:00:46,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e6d31893a761fb01419b5d3147eeec1a in 114ms, sequenceid=194, compaction requested=false 2024-11-16T00:00:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T00:00:46,530 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:46,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T00:00:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:46,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:46,533 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:46,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:46,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:46,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:00:46,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/996bb9b689814cc0843d72fc74a7f71f is 50, key is test_row_0/A:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742341_1517 (size=16931) 2024-11-16T00:00:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715306606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715306608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715306609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715306614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:46,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715306699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715306720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715306733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715306735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715306736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:46,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:46,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:46,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715306940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715306940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715306940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715306950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:46,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/996bb9b689814cc0843d72fc74a7f71f 2024-11-16T00:00:46,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b145a367b9184449a1fccb7c84a91747 is 50, key is test_row_0/B:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:47,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742342_1518 (size=12151) 2024-11-16T00:00:47,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b145a367b9184449a1fccb7c84a91747 2024-11-16T00:00:47,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ea0a67f156554dea85a92e41360614d0 is 50, key is test_row_0/C:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:47,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742343_1519 (size=12151) 2024-11-16T00:00:47,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:47,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715307260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715307260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715307260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715307269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:47,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:47,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:47,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ea0a67f156554dea85a92e41360614d0 2024-11-16T00:00:47,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/996bb9b689814cc0843d72fc74a7f71f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f 2024-11-16T00:00:47,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f, entries=250, sequenceid=213, filesize=16.5 K 2024-11-16T00:00:47,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b145a367b9184449a1fccb7c84a91747 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747 2024-11-16T00:00:47,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T00:00:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ea0a67f156554dea85a92e41360614d0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0 2024-11-16T00:00:47,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T00:00:47,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e6d31893a761fb01419b5d3147eeec1a in 975ms, sequenceid=213, compaction requested=true 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:47,539 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T00:00:47,539 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:47,541 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:47,541 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:47,541 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,541 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a552d9be4b9140f98b8d9e832dca5ee9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.0 K 2024-11-16T00:00:47,543 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting a552d9be4b9140f98b8d9e832dca5ee9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:47,543 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702d87d41b7249c093619ade5fa97ddb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731715245718 2024-11-16T00:00:47,543 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b145a367b9184449a1fccb7c84a91747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246447 2024-11-16T00:00:47,546 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:47,546 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:47,546 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,546 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/bd8b1bf12906429dbd13757042e99071, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=43.0 K 2024-11-16T00:00:47,546 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8b1bf12906429dbd13757042e99071, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:47,549 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 73df2f06cc2c4e0882093135e99c3eba, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731715245718 2024-11-16T00:00:47,550 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 996bb9b689814cc0843d72fc74a7f71f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246423 2024-11-16T00:00:47,567 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:47,567 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:47,568 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/29e8cef55cc440e5bb7cef6b78f7e689 is 50, key is test_row_0/A:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:47,568 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/64a06ad131e14fa4a39510ebbf8a6861 is 50, key is test_row_0/B:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:47,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742344_1520 (size=12663) 2024-11-16T00:00:47,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742345_1521 (size=12663) 2024-11-16T00:00:47,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T00:00:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:00:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/25848b44d9b14eadb3c14cf461ec40fd is 50, key is test_row_0/A:col10/1731715246599/Put/seqid=0 2024-11-16T00:00:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742346_1522 (size=12151) 2024-11-16T00:00:47,678 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/25848b44d9b14eadb3c14cf461ec40fd 2024-11-16T00:00:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/321427ec0a744256861a48e40596f299 is 50, key is test_row_0/B:col10/1731715246599/Put/seqid=0 2024-11-16T00:00:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742347_1523 (size=12151) 2024-11-16T00:00:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:47,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715307810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715307852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715307852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715307852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715307853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715307970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715307980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715307980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715307980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715307980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,022 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/29e8cef55cc440e5bb7cef6b78f7e689 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/29e8cef55cc440e5bb7cef6b78f7e689 2024-11-16T00:00:48,026 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 29e8cef55cc440e5bb7cef6b78f7e689(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:48,027 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,027 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715247539; duration=0sec 2024-11-16T00:00:48,027 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:48,027 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:48,027 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:48,028 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:48,028 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:48,028 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,028 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/adba1bc4ff9a47a4839929792e5bc60a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.0 K 2024-11-16T00:00:48,029 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting adba1bc4ff9a47a4839929792e5bc60a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731715245528 2024-11-16T00:00:48,029 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7649f3898704409695fb587cee35eb75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731715245718 2024-11-16T00:00:48,029 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/64a06ad131e14fa4a39510ebbf8a6861 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/64a06ad131e14fa4a39510ebbf8a6861 2024-11-16T00:00:48,030 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0a67f156554dea85a92e41360614d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246447 2024-11-16T00:00:48,041 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into 64a06ad131e14fa4a39510ebbf8a6861(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:48,041 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,041 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715247539; duration=0sec 2024-11-16T00:00:48,042 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:48,042 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:48,060 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:48,061 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0b2120165d7b44a3b56369eb4be01d9d is 50, key is test_row_0/C:col10/1731715246563/Put/seqid=0 2024-11-16T00:00:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742348_1524 (size=12663) 2024-11-16T00:00:48,107 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/321427ec0a744256861a48e40596f299 2024-11-16T00:00:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0afe87c2a9914a94b6b9cafe6228e52c is 50, key is test_row_0/C:col10/1731715246599/Put/seqid=0 2024-11-16T00:00:48,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742349_1525 (size=12151) 2024-11-16T00:00:48,127 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0afe87c2a9914a94b6b9cafe6228e52c 2024-11-16T00:00:48,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/25848b44d9b14eadb3c14cf461ec40fd as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd 2024-11-16T00:00:48,138 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T00:00:48,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/321427ec0a744256861a48e40596f299 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299 2024-11-16T00:00:48,144 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T00:00:48,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0afe87c2a9914a94b6b9cafe6228e52c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c 2024-11-16T00:00:48,148 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T00:00:48,149 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e6d31893a761fb01419b5d3147eeec1a in 513ms, sequenceid=230, compaction requested=false 2024-11-16T00:00:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T00:00:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T00:00:48,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T00:00:48,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-16T00:00:48,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6220 sec 2024-11-16T00:00:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:48,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:48,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:00:48,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/63bbec44e8aa4e46a9a2350d60b87aed is 50, key is test_row_0/A:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742350_1526 (size=14541) 2024-11-16T00:00:48,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/63bbec44e8aa4e46a9a2350d60b87aed 2024-11-16T00:00:48,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d4feeb9637254bd3b1d11e02b78323f0 is 50, key is test_row_0/B:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742351_1527 (size=12151) 2024-11-16T00:00:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715308230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715308232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715308233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715308241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715308243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715308360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715308360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715308360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715308361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715308370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,489 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/0b2120165d7b44a3b56369eb4be01d9d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0b2120165d7b44a3b56369eb4be01d9d 2024-11-16T00:00:48,497 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 0b2120165d7b44a3b56369eb4be01d9d(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:48,497 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,497 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715247539; duration=0sec 2024-11-16T00:00:48,498 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:48,498 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:48,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715308579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715308579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715308580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715308580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715308590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d4feeb9637254bd3b1d11e02b78323f0 2024-11-16T00:00:48,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/352d98be46aa41cab7e1c6a468be8e2b is 50, key is test_row_0/C:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742352_1528 (size=12151) 2024-11-16T00:00:48,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/352d98be46aa41cab7e1c6a468be8e2b 2024-11-16T00:00:48,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/63bbec44e8aa4e46a9a2350d60b87aed as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed 2024-11-16T00:00:48,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed, entries=200, sequenceid=252, filesize=14.2 K 2024-11-16T00:00:48,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d4feeb9637254bd3b1d11e02b78323f0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0 2024-11-16T00:00:48,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T00:00:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/352d98be46aa41cab7e1c6a468be8e2b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b 2024-11-16T00:00:48,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T00:00:48,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e6d31893a761fb01419b5d3147eeec1a in 477ms, sequenceid=252, compaction requested=true 2024-11-16T00:00:48,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:48,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:48,675 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:48,675 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:48,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:48,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:48,676 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,676 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/29e8cef55cc440e5bb7cef6b78f7e689, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=38.4 K 2024-11-16T00:00:48,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:48,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:48,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:48,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:48,680 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:48,680 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:48,680 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e8cef55cc440e5bb7cef6b78f7e689, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246447 2024-11-16T00:00:48,680 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,680 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/64a06ad131e14fa4a39510ebbf8a6861, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.1 K 2024-11-16T00:00:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T00:00:48,680 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a06ad131e14fa4a39510ebbf8a6861, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246447 2024-11-16T00:00:48,680 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:48,681 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25848b44d9b14eadb3c14cf461ec40fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731715246599 2024-11-16T00:00:48,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:48,682 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 321427ec0a744256861a48e40596f299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731715246599 2024-11-16T00:00:48,682 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63bbec44e8aa4e46a9a2350d60b87aed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:48,682 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d4feeb9637254bd3b1d11e02b78323f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T00:00:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T00:00:48,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:48,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:48,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:48,699 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:48,700 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/06f50ba3049743dab02bc2220610e554 is 50, key is test_row_0/A:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,705 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:48,706 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d924c03ceac1410a8f15c20acda876bf is 50, key is test_row_0/B:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742353_1529 (size=12765) 2024-11-16T00:00:48,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742354_1530 (size=12765) 2024-11-16T00:00:48,720 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d924c03ceac1410a8f15c20acda876bf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d924c03ceac1410a8f15c20acda876bf 2024-11-16T00:00:48,725 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into d924c03ceac1410a8f15c20acda876bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:48,725 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,725 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715248675; duration=0sec 2024-11-16T00:00:48,726 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:48,726 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:48,726 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:48,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:48,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:48,727 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,727 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0b2120165d7b44a3b56369eb4be01d9d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.1 K 2024-11-16T00:00:48,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2120165d7b44a3b56369eb4be01d9d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731715246447 2024-11-16T00:00:48,727 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0afe87c2a9914a94b6b9cafe6228e52c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731715246599 2024-11-16T00:00:48,728 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 352d98be46aa41cab7e1c6a468be8e2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:48,748 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:48,748 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/76084f767bd84bc29ab6dded27d88526 is 50, key is test_row_0/C:col10/1731715247812/Put/seqid=0 2024-11-16T00:00:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742355_1531 (size=12765) 2024-11-16T00:00:48,764 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/76084f767bd84bc29ab6dded27d88526 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/76084f767bd84bc29ab6dded27d88526 2024-11-16T00:00:48,771 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 76084f767bd84bc29ab6dded27d88526(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:48,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,771 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715248678; duration=0sec 2024-11-16T00:00:48,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:48,771 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T00:00:48,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T00:00:48,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,838 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T00:00:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/cbfff99e1d474d3a8197d3bdcbfd20e7 is 50, key is test_row_0/A:col10/1731715248230/Put/seqid=0 2024-11-16T00:00:48,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742356_1532 (size=12301) 2024-11-16T00:00:48,878 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/cbfff99e1d474d3a8197d3bdcbfd20e7 2024-11-16T00:00:48,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/13a424ca27a540d1a135d5b6054a72e2 is 50, key is test_row_0/B:col10/1731715248230/Put/seqid=0 2024-11-16T00:00:48,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:48,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742357_1533 (size=12301) 2024-11-16T00:00:48,920 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/13a424ca27a540d1a135d5b6054a72e2 2024-11-16T00:00:48,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/95955f4274954f1bac559d7aa0d3f471 is 50, key is test_row_0/C:col10/1731715248230/Put/seqid=0 2024-11-16T00:00:48,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742358_1534 (size=12301) 2024-11-16T00:00:48,959 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/95955f4274954f1bac559d7aa0d3f471 2024-11-16T00:00:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715308945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715308946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715308954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715308954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/cbfff99e1d474d3a8197d3bdcbfd20e7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7 2024-11-16T00:00:48,968 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T00:00:48,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/13a424ca27a540d1a135d5b6054a72e2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2 2024-11-16T00:00:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715308955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:48,977 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T00:00:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/95955f4274954f1bac559d7aa0d3f471 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471 2024-11-16T00:00:48,982 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T00:00:48,983 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e6d31893a761fb01419b5d3147eeec1a in 146ms, sequenceid=271, compaction requested=false 2024-11-16T00:00:48,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:48,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:48,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T00:00:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T00:00:48,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T00:00:48,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 300 msec 2024-11-16T00:00:48,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 306 msec 2024-11-16T00:00:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T00:00:49,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:49,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T00:00:49,003 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:49,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:49,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:49,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T00:00:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:49,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/941decf7722742bab4bdb820e1aa0bf7 is 50, key is test_row_0/A:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742359_1535 (size=12301) 2024-11-16T00:00:49,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/941decf7722742bab4bdb820e1aa0bf7 2024-11-16T00:00:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:49,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/14fcca67a48d472c8b9653622da3a26e is 50, key is test_row_0/B:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,123 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/06f50ba3049743dab02bc2220610e554 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/06f50ba3049743dab02bc2220610e554 2024-11-16T00:00:49,127 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 06f50ba3049743dab02bc2220610e554(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:49,127 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,127 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715248674; duration=0sec 2024-11-16T00:00:49,127 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:49,127 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715309112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715309113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742360_1536 (size=12301) 2024-11-16T00:00:49,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/14fcca67a48d472c8b9653622da3a26e 2024-11-16T00:00:49,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1dc3569bf46545df96e9616d641730de is 50, key is test_row_0/C:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715309119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715309130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715309130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742361_1537 (size=12301) 2024-11-16T00:00:49,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T00:00:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1dc3569bf46545df96e9616d641730de 2024-11-16T00:00:49,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/941decf7722742bab4bdb820e1aa0bf7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7 2024-11-16T00:00:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T00:00:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/14fcca67a48d472c8b9653622da3a26e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e 2024-11-16T00:00:49,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T00:00:49,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1dc3569bf46545df96e9616d641730de as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de 2024-11-16T00:00:49,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T00:00:49,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for e6d31893a761fb01419b5d3147eeec1a in 104ms, sequenceid=293, compaction requested=true 2024-11-16T00:00:49,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,185 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:49,185 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:49,185 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:49,185 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,186 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/06f50ba3049743dab02bc2220610e554, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.5 K 2024-11-16T00:00:49,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:49,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:49,186 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f50ba3049743dab02bc2220610e554, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:49,186 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbfff99e1d474d3a8197d3bdcbfd20e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731715248230 2024-11-16T00:00:49,186 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941decf7722742bab4bdb820e1aa0bf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:49,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:49,188 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:49,189 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:49,189 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,189 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d924c03ceac1410a8f15c20acda876bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.5 K 2024-11-16T00:00:49,189 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d924c03ceac1410a8f15c20acda876bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:49,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a424ca27a540d1a135d5b6054a72e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731715248230 2024-11-16T00:00:49,190 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 14fcca67a48d472c8b9653622da3a26e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:49,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:49,196 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:49,197 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/693833e9257f4de2ae971a110bfd54ed is 50, key is test_row_0/A:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,207 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:49,208 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a062011f7dca4c65beadf1443b3303e9 is 50, key is test_row_0/B:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742362_1538 (size=13017) 2024-11-16T00:00:49,214 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/693833e9257f4de2ae971a110bfd54ed as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/693833e9257f4de2ae971a110bfd54ed 2024-11-16T00:00:49,223 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 693833e9257f4de2ae971a110bfd54ed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:49,223 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,223 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=13, startTime=1731715249184; duration=0sec 2024-11-16T00:00:49,223 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:49,223 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:49,223 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:00:49,224 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:00:49,224 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:49,224 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,224 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/76084f767bd84bc29ab6dded27d88526, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=36.5 K 2024-11-16T00:00:49,224 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76084f767bd84bc29ab6dded27d88526, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731715247808 2024-11-16T00:00:49,224 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95955f4274954f1bac559d7aa0d3f471, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731715248230 2024-11-16T00:00:49,225 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc3569bf46545df96e9616d641730de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:49,234 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:49,235 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/da988b551eb848f085be05b106593139 is 50, key is test_row_0/C:col10/1731715249080/Put/seqid=0 2024-11-16T00:00:49,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742363_1539 (size=13017) 2024-11-16T00:00:49,243 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/a062011f7dca4c65beadf1443b3303e9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a062011f7dca4c65beadf1443b3303e9 2024-11-16T00:00:49,250 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into a062011f7dca4c65beadf1443b3303e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:49,250 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,250 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=13, startTime=1731715249187; duration=0sec 2024-11-16T00:00:49,250 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:49,250 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:49,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T00:00:49,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/5b9fd7f1e5504e849eccdf854fb99d63 is 50, key is test_row_0/A:col10/1731715249128/Put/seqid=0 2024-11-16T00:00:49,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742364_1540 (size=13017) 2024-11-16T00:00:49,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742365_1541 (size=14741) 2024-11-16T00:00:49,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715309287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715309289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715309289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715309298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715309299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T00:00:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715309410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715309410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715309410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715309420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715309420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T00:00:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T00:00:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:00:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715309621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715309621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:49,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715309621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715309631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715309640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:49,671 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/da988b551eb848f085be05b106593139 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/da988b551eb848f085be05b106593139 2024-11-16T00:00:49,675 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into da988b551eb848f085be05b106593139(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:49,675 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,675 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=13, startTime=1731715249191; duration=0sec 2024-11-16T00:00:49,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:49,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:49,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/5b9fd7f1e5504e849eccdf854fb99d63 2024-11-16T00:00:49,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/0c106d40c07c41d5806ac07db7678b21 is 50, key is test_row_0/B:col10/1731715249128/Put/seqid=0 2024-11-16T00:00:49,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742366_1542 (size=12301) 2024-11-16T00:00:49,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/0c106d40c07c41d5806ac07db7678b21 2024-11-16T00:00:49,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d7ccc543865840a29b28657b3d50704a is 50, key is test_row_0/C:col10/1731715249128/Put/seqid=0 2024-11-16T00:00:49,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742367_1543 (size=12301) 2024-11-16T00:00:49,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d7ccc543865840a29b28657b3d50704a 2024-11-16T00:00:49,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/5b9fd7f1e5504e849eccdf854fb99d63 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63 2024-11-16T00:00:49,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63, entries=200, sequenceid=313, filesize=14.4 K 2024-11-16T00:00:49,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/0c106d40c07c41d5806ac07db7678b21 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21 2024-11-16T00:00:49,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T00:00:49,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/d7ccc543865840a29b28657b3d50704a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a 2024-11-16T00:00:49,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T00:00:49,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e6d31893a761fb01419b5d3147eeec1a in 488ms, sequenceid=313, compaction requested=false 2024-11-16T00:00:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:49,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T00:00:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:49,774 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:00:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/8cdd5ba00ece4c72b80af77339affe1e is 50, key is test_row_0/A:col10/1731715249297/Put/seqid=0 2024-11-16T00:00:49,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742368_1544 (size=12301) 2024-11-16T00:00:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:49,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:50,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715309981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715309983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715310013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715310013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715310013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715310120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715310120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:50,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715310140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715310141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715310141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,208 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/8cdd5ba00ece4c72b80af77339affe1e 2024-11-16T00:00:50,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/6564e5b4edf64f5f901f71e43d656268 is 50, key is test_row_0/B:col10/1731715249297/Put/seqid=0 2024-11-16T00:00:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742369_1545 (size=12301) 2024-11-16T00:00:50,313 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/6564e5b4edf64f5f901f71e43d656268 2024-11-16T00:00:50,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715310341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715310341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/bc9bd1be58f34af7ac00e555a6d87525 is 50, key is test_row_0/C:col10/1731715249297/Put/seqid=0 2024-11-16T00:00:50,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715310361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715310365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715310365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742370_1546 (size=12301) 2024-11-16T00:00:50,402 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/bc9bd1be58f34af7ac00e555a6d87525 2024-11-16T00:00:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/8cdd5ba00ece4c72b80af77339affe1e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e 2024-11-16T00:00:50,427 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T00:00:50,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/6564e5b4edf64f5f901f71e43d656268 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268 2024-11-16T00:00:50,431 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T00:00:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/bc9bd1be58f34af7ac00e555a6d87525 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525 2024-11-16T00:00:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525, entries=150, sequenceid=333, filesize=12.0 K 2024-11-16T00:00:50,437 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for e6d31893a761fb01419b5d3147eeec1a in 663ms, sequenceid=333, compaction requested=true 2024-11-16T00:00:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T00:00:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T00:00:50,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T00:00:50,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-16T00:00:50,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4410 sec 2024-11-16T00:00:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:50,667 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:50,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:00:50,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/20de24af87514e6d9db777f3ffdc36f6 is 50, key is test_row_0/A:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:50,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742371_1547 (size=14741) 2024-11-16T00:00:50,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715310704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715310705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715310746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715310746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715310746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715310850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715310849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715310870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715310870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715310870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/20de24af87514e6d9db777f3ffdc36f6 2024-11-16T00:00:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715311070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715311070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b8d47749ba45414cba96ae3651f79543 is 50, key is test_row_0/B:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:51,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742372_1548 (size=12301) 2024-11-16T00:00:51,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b8d47749ba45414cba96ae3651f79543 2024-11-16T00:00:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715311089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715311089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715311090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f08a08d454524ae380ccf48322196aa1 is 50, key is test_row_0/C:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:51,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742373_1549 (size=12301) 2024-11-16T00:00:51,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f08a08d454524ae380ccf48322196aa1 2024-11-16T00:00:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/20de24af87514e6d9db777f3ffdc36f6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6 2024-11-16T00:00:51,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6, entries=200, sequenceid=353, filesize=14.4 K 2024-11-16T00:00:51,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/b8d47749ba45414cba96ae3651f79543 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543 2024-11-16T00:00:51,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T00:00:51,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/f08a08d454524ae380ccf48322196aa1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1 2024-11-16T00:00:51,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T00:00:51,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 457ms, sequenceid=353, compaction requested=true 2024-11-16T00:00:51,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:51,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:51,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:51,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:51,125 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:51,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:51,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:51,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:51,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:51,131 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:51,131 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:51,131 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:51,131 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a062011f7dca4c65beadf1443b3303e9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=48.8 K 2024-11-16T00:00:51,131 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:51,131 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:51,131 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:51,131 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a062011f7dca4c65beadf1443b3303e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:51,131 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/693833e9257f4de2ae971a110bfd54ed, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=53.5 K 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c106d40c07c41d5806ac07db7678b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715249128 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693833e9257f4de2ae971a110bfd54ed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 6564e5b4edf64f5f901f71e43d656268, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715249285 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9fd7f1e5504e849eccdf854fb99d63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715249128 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d47749ba45414cba96ae3651f79543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249992 2024-11-16T00:00:51,132 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdd5ba00ece4c72b80af77339affe1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715249285 2024-11-16T00:00:51,133 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20de24af87514e6d9db777f3ffdc36f6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249979 2024-11-16T00:00:51,144 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:51,144 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/241eb907f83f452c93a338af9212ba3e is 50, key is test_row_0/B:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:51,148 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:51,149 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/f1eac32e0970455b81735c1b76d8a08d is 50, key is test_row_0/A:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T00:00:51,150 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:51,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T00:00:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T00:00:51,153 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:51,154 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:51,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:51,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742375_1551 (size=13153) 2024-11-16T00:00:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742374_1550 (size=13153) 2024-11-16T00:00:51,171 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/f1eac32e0970455b81735c1b76d8a08d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f1eac32e0970455b81735c1b76d8a08d 2024-11-16T00:00:51,177 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/241eb907f83f452c93a338af9212ba3e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/241eb907f83f452c93a338af9212ba3e 2024-11-16T00:00:51,177 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into f1eac32e0970455b81735c1b76d8a08d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:51,177 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:51,177 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=12, startTime=1731715251124; duration=0sec 2024-11-16T00:00:51,178 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:51,178 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:51,178 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:51,180 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:51,180 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:51,180 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:51,180 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/da988b551eb848f085be05b106593139, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=48.8 K 2024-11-16T00:00:51,182 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting da988b551eb848f085be05b106593139, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731715248952 2024-11-16T00:00:51,184 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7ccc543865840a29b28657b3d50704a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731715249128 2024-11-16T00:00:51,185 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9bd1be58f34af7ac00e555a6d87525, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731715249285 2024-11-16T00:00:51,186 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08a08d454524ae380ccf48322196aa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249992 2024-11-16T00:00:51,187 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into 241eb907f83f452c93a338af9212ba3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:51,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:51,187 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=12, startTime=1731715251125; duration=0sec 2024-11-16T00:00:51,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:51,187 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:51,208 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:51,208 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/180c5d6677d64a18a96169414eee79db is 50, key is test_row_0/C:col10/1731715250666/Put/seqid=0 2024-11-16T00:00:51,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742376_1552 (size=13153) 2024-11-16T00:00:51,226 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/180c5d6677d64a18a96169414eee79db as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/180c5d6677d64a18a96169414eee79db 2024-11-16T00:00:51,233 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 180c5d6677d64a18a96169414eee79db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:51,233 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:51,233 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=12, startTime=1731715251126; duration=0sec 2024-11-16T00:00:51,233 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:51,233 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T00:00:51,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T00:00:51,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:51,307 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:00:51,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ad34d7d41d75407b87bc37b563355858 is 50, key is test_row_0/A:col10/1731715250701/Put/seqid=0 2024-11-16T00:00:51,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742377_1553 (size=12301) 2024-11-16T00:00:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:51,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715311420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715311423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715311425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715311427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715311433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T00:00:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715311540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715311540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715311540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715311550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715311550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,738 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ad34d7d41d75407b87bc37b563355858 2024-11-16T00:00:51,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/5f5425cd8330410ea18ccd938609b6d1 is 50, key is test_row_0/B:col10/1731715250701/Put/seqid=0 2024-11-16T00:00:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715311750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742378_1554 (size=12301) 2024-11-16T00:00:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715311750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715311750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715311760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715311760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T00:00:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715312070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715312070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715312070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715312080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715312080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,159 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/5f5425cd8330410ea18ccd938609b6d1 2024-11-16T00:00:52,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8e6d236556f44fc9905e9a62cf41ea42 is 50, key is test_row_0/C:col10/1731715250701/Put/seqid=0 2024-11-16T00:00:52,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742379_1555 (size=12301) 2024-11-16T00:00:52,176 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8e6d236556f44fc9905e9a62cf41ea42 2024-11-16T00:00:52,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ad34d7d41d75407b87bc37b563355858 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858 2024-11-16T00:00:52,189 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T00:00:52,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/5f5425cd8330410ea18ccd938609b6d1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1 2024-11-16T00:00:52,194 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T00:00:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8e6d236556f44fc9905e9a62cf41ea42 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42 2024-11-16T00:00:52,203 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42, entries=150, sequenceid=374, filesize=12.0 K 2024-11-16T00:00:52,205 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 898ms, sequenceid=374, compaction requested=false 2024-11-16T00:00:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T00:00:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T00:00:52,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T00:00:52,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0530 sec 2024-11-16T00:00:52,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0590 sec 2024-11-16T00:00:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T00:00:52,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:52,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T00:00:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T00:00:52,300 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:52,301 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:52,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T00:00:52,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T00:00:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:52,454 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:00:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0bb9276241c5498e839ebada45de8394 is 50, key is test_row_0/A:col10/1731715251410/Put/seqid=0 2024-11-16T00:00:52,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742380_1556 (size=12301) 2024-11-16T00:00:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T00:00:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715312620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715312618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715312625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715312634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715312635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715312740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715312740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715312741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715312760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715312760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:52,895 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0bb9276241c5498e839ebada45de8394 2024-11-16T00:00:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/7674d6175e4f426c9fb63136f8e1c398 is 50, key is test_row_0/B:col10/1731715251410/Put/seqid=0 2024-11-16T00:00:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742381_1557 (size=12301) 2024-11-16T00:00:52,908 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/7674d6175e4f426c9fb63136f8e1c398 2024-11-16T00:00:52,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/994ddc5cb03a474483ae955262a2511a is 50, key is test_row_0/C:col10/1731715251410/Put/seqid=0 2024-11-16T00:00:52,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742382_1558 (size=12301) 2024-11-16T00:00:52,918 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/994ddc5cb03a474483ae955262a2511a 2024-11-16T00:00:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/0bb9276241c5498e839ebada45de8394 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394 2024-11-16T00:00:52,927 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T00:00:52,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/7674d6175e4f426c9fb63136f8e1c398 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398 2024-11-16T00:00:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T00:00:52,931 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T00:00:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/994ddc5cb03a474483ae955262a2511a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a 2024-11-16T00:00:52,934 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T00:00:52,935 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 481ms, sequenceid=392, compaction requested=true 2024-11-16T00:00:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T00:00:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T00:00:52,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T00:00:52,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 635 msec 2024-11-16T00:00:52,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 641 msec 2024-11-16T00:00:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:52,966 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:52,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:00:52,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/85426e9e42644bfc8654e839dac0dafc is 50, key is test_row_0/A:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:52,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742383_1559 (size=17181) 2024-11-16T00:00:52,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/85426e9e42644bfc8654e839dac0dafc 2024-11-16T00:00:52,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 is 50, key is test_row_0/B:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:53,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742384_1560 (size=12301) 2024-11-16T00:00:53,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 2024-11-16T00:00:53,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8efe31e5f35e4497a79c58110673f117 is 50, key is test_row_0/C:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:53,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715312997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742385_1561 (size=12301) 2024-11-16T00:00:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715313009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715313011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715313011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715313013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715313121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715313339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715313350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715313350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715313350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715313350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8efe31e5f35e4497a79c58110673f117 2024-11-16T00:00:53,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/85426e9e42644bfc8654e839dac0dafc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc 2024-11-16T00:00:53,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc, entries=250, sequenceid=412, filesize=16.8 K 2024-11-16T00:00:53,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 2024-11-16T00:00:53,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5, entries=150, sequenceid=412, filesize=12.0 K 2024-11-16T00:00:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/8efe31e5f35e4497a79c58110673f117 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117 2024-11-16T00:00:53,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117, entries=150, sequenceid=412, filesize=12.0 K 2024-11-16T00:00:53,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e6d31893a761fb01419b5d3147eeec1a in 471ms, sequenceid=412, compaction requested=true 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:53,437 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:53,437 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:53,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T00:00:53,440 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:53,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-16T00:00:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T00:00:53,442 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:53,443 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:53,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:53,448 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:53,448 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:53,448 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:53,448 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:53,448 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:53,448 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:53,448 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f1eac32e0970455b81735c1b76d8a08d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=53.6 K 2024-11-16T00:00:53,448 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/241eb907f83f452c93a338af9212ba3e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=48.9 K 2024-11-16T00:00:53,449 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1eac32e0970455b81735c1b76d8a08d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249992 2024-11-16T00:00:53,449 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 241eb907f83f452c93a338af9212ba3e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249992 2024-11-16T00:00:53,450 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5425cd8330410ea18ccd938609b6d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731715250701 2024-11-16T00:00:53,450 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad34d7d41d75407b87bc37b563355858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731715250701 2024-11-16T00:00:53,451 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb9276241c5498e839ebada45de8394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731715251410 2024-11-16T00:00:53,451 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7674d6175e4f426c9fb63136f8e1c398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731715251410 2024-11-16T00:00:53,451 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a89cc0478424ab2a1ec2aeb4fdb85a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252632 2024-11-16T00:00:53,451 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85426e9e42644bfc8654e839dac0dafc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252618 2024-11-16T00:00:53,464 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:53,465 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/e1147271881c4684ac07586edc151ce3 is 50, key is test_row_0/B:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:53,476 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:53,476 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/5838b8048c354e0c97523071f37fac70 is 50, key is test_row_0/A:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742386_1562 (size=13289) 2024-11-16T00:00:53,506 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/e1147271881c4684ac07586edc151ce3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/e1147271881c4684ac07586edc151ce3 2024-11-16T00:00:53,512 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into e1147271881c4684ac07586edc151ce3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:53,512 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:53,512 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=12, startTime=1731715253437; duration=0sec 2024-11-16T00:00:53,512 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:53,512 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:53,512 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742387_1563 (size=13289) 2024-11-16T00:00:53,513 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:53,514 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:53,514 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:53,514 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/180c5d6677d64a18a96169414eee79db, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=48.9 K 2024-11-16T00:00:53,515 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 180c5d6677d64a18a96169414eee79db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731715249992 2024-11-16T00:00:53,516 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6d236556f44fc9905e9a62cf41ea42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731715250701 2024-11-16T00:00:53,517 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 994ddc5cb03a474483ae955262a2511a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731715251410 2024-11-16T00:00:53,519 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 8efe31e5f35e4497a79c58110673f117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252632 2024-11-16T00:00:53,520 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/5838b8048c354e0c97523071f37fac70 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5838b8048c354e0c97523071f37fac70 2024-11-16T00:00:53,528 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 5838b8048c354e0c97523071f37fac70(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:53,528 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:53,528 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=12, startTime=1731715253437; duration=0sec 2024-11-16T00:00:53,529 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:53,529 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:53,532 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:53,533 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/80cc13e8bb4b44faa4912b4ccf2e737b is 50, key is test_row_0/C:col10/1731715252964/Put/seqid=0 2024-11-16T00:00:53,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742388_1564 (size=13289) 2024-11-16T00:00:53,541 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/80cc13e8bb4b44faa4912b4ccf2e737b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/80cc13e8bb4b44faa4912b4ccf2e737b 2024-11-16T00:00:53,545 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 80cc13e8bb4b44faa4912b4ccf2e737b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:53,545 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:53,545 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=12, startTime=1731715253437; duration=0sec 2024-11-16T00:00:53,545 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:53,545 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T00:00:53,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T00:00:53,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:53,597 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T00:00:53,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/1a7ad360d9ee4fd8ae26b5ca5f91768c is 50, key is test_row_0/A:col10/1731715253010/Put/seqid=0 2024-11-16T00:00:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742389_1565 (size=12301) 2024-11-16T00:00:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:53,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715313699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715313710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715313711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715313712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715313715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T00:00:53,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715313820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715313820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715313830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715313830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:53,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715313839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,018 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/1a7ad360d9ee4fd8ae26b5ca5f91768c 2024-11-16T00:00:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/3788af4bd80a4b58b94bd0ea4c138c13 is 50, key is test_row_0/B:col10/1731715253010/Put/seqid=0 2024-11-16T00:00:54,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742390_1566 (size=12301) 2024-11-16T00:00:54,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715314040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715314040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715314050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715314050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715314050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T00:00:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715314360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715314360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715314369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715314370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715314370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,441 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/3788af4bd80a4b58b94bd0ea4c138c13 2024-11-16T00:00:54,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3d6b1d7a1a8647ad887451d7a4ee54c9 is 50, key is test_row_0/C:col10/1731715253010/Put/seqid=0 2024-11-16T00:00:54,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742391_1567 (size=12301) 2024-11-16T00:00:54,454 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3d6b1d7a1a8647ad887451d7a4ee54c9 2024-11-16T00:00:54,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/1a7ad360d9ee4fd8ae26b5ca5f91768c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c 2024-11-16T00:00:54,464 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T00:00:54,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/3788af4bd80a4b58b94bd0ea4c138c13 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13 2024-11-16T00:00:54,468 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T00:00:54,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3d6b1d7a1a8647ad887451d7a4ee54c9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9 2024-11-16T00:00:54,475 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T00:00:54,476 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e6d31893a761fb01419b5d3147eeec1a in 880ms, sequenceid=431, compaction requested=false 2024-11-16T00:00:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-16T00:00:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-16T00:00:54,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T00:00:54,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-16T00:00:54,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0390 sec 2024-11-16T00:00:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T00:00:54,580 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:54,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-16T00:00:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T00:00:54,583 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:54,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:54,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T00:00:54,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T00:00:54,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:54,737 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:00:54,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4b000f02a21048e8925f8a161e618f2b is 50, key is test_row_0/A:col10/1731715253708/Put/seqid=0 2024-11-16T00:00:54,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742392_1568 (size=12301) 2024-11-16T00:00:54,746 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4b000f02a21048e8925f8a161e618f2b 2024-11-16T00:00:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/c37badaa4fe54f61b24669811a4a2a67 is 50, key is test_row_0/B:col10/1731715253708/Put/seqid=0 2024-11-16T00:00:54,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742393_1569 (size=12301) 2024-11-16T00:00:54,769 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/c37badaa4fe54f61b24669811a4a2a67 2024-11-16T00:00:54,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3ed2b75915024475a4569163381dc44b is 50, key is test_row_0/C:col10/1731715253708/Put/seqid=0 2024-11-16T00:00:54,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742394_1570 (size=12301) 2024-11-16T00:00:54,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T00:00:54,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715314914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715314915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715314923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715314926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715314927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715315030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715315040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715315050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715315050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715315050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,186 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3ed2b75915024475a4569163381dc44b 2024-11-16T00:00:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/4b000f02a21048e8925f8a161e618f2b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b 2024-11-16T00:00:55,194 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T00:00:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/c37badaa4fe54f61b24669811a4a2a67 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67 2024-11-16T00:00:55,199 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T00:00:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3ed2b75915024475a4569163381dc44b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b 2024-11-16T00:00:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T00:00:55,222 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T00:00:55,224 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e6d31893a761fb01419b5d3147eeec1a in 486ms, sequenceid=451, compaction requested=true 2024-11-16T00:00:55,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:55,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:55,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-16T00:00:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-16T00:00:55,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T00:00:55,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2024-11-16T00:00:55,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 648 msec 2024-11-16T00:00:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:55,254 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:55,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:00:55,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ae8f786931c84ad18a7ce67cc7885588 is 50, key is test_row_0/A:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742395_1571 (size=14741) 2024-11-16T00:00:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715315288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715315300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715315303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715315304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715315305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715315409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715315420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715315420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715315420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715315420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715315630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715315640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715315640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715315640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715315640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:55,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ae8f786931c84ad18a7ce67cc7885588 2024-11-16T00:00:55,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/445725df1dd0440f978ce0c77097af22 is 50, key is test_row_0/B:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742396_1572 (size=12301) 2024-11-16T00:00:55,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/445725df1dd0440f978ce0c77097af22 2024-11-16T00:00:55,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2a8122668980482f8272d2616e27fed5 is 50, key is test_row_0/C:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T00:00:55,720 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:55,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-16T00:00:55,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742397_1573 (size=12301) 2024-11-16T00:00:55,722 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T00:00:55,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2a8122668980482f8272d2616e27fed5 2024-11-16T00:00:55,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:55,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:55,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/ae8f786931c84ad18a7ce67cc7885588 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588 2024-11-16T00:00:55,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588, entries=200, sequenceid=468, filesize=14.4 K 2024-11-16T00:00:55,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/445725df1dd0440f978ce0c77097af22 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22 2024-11-16T00:00:55,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22, entries=150, sequenceid=468, filesize=12.0 K 2024-11-16T00:00:55,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/2a8122668980482f8272d2616e27fed5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5 2024-11-16T00:00:55,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5, entries=150, sequenceid=468, filesize=12.0 K 2024-11-16T00:00:55,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e6d31893a761fb01419b5d3147eeec1a in 496ms, sequenceid=468, compaction requested=true 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:55,750 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:55,750 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:00:55,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:55,759 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:55,759 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/B is initiating minor compaction (all files) 2024-11-16T00:00:55,759 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/B in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:55,759 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/e1147271881c4684ac07586edc151ce3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=49.0 K 2024-11-16T00:00:55,759 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:55,759 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/A is initiating minor compaction (all files) 2024-11-16T00:00:55,759 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/A in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:55,759 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5838b8048c354e0c97523071f37fac70, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=51.4 K 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e1147271881c4684ac07586edc151ce3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252632 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5838b8048c354e0c97523071f37fac70, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252632 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3788af4bd80a4b58b94bd0ea4c138c13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731715253006 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7ad360d9ee4fd8ae26b5ca5f91768c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731715253006 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c37badaa4fe54f61b24669811a4a2a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731715253695 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b000f02a21048e8925f8a161e618f2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731715253695 2024-11-16T00:00:55,760 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 445725df1dd0440f978ce0c77097af22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731715254909 2024-11-16T00:00:55,761 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae8f786931c84ad18a7ce67cc7885588, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731715254909 2024-11-16T00:00:55,772 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:55,772 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/ee40bc54827c403db401fb80bc30f73c is 50, key is test_row_0/B:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,779 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#A#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:55,779 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/86b85faf03584c89a5d4248b1c043826 is 50, key is test_row_0/A:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742398_1574 (size=13425) 2024-11-16T00:00:55,805 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/ee40bc54827c403db401fb80bc30f73c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/ee40bc54827c403db401fb80bc30f73c 2024-11-16T00:00:55,810 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/B of e6d31893a761fb01419b5d3147eeec1a into ee40bc54827c403db401fb80bc30f73c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:55,810 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:55,810 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/B, priority=12, startTime=1731715255750; duration=0sec 2024-11-16T00:00:55,810 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:00:55,810 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:55,810 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:00:55,811 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:00:55,811 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): e6d31893a761fb01419b5d3147eeec1a/C is initiating minor compaction (all files) 2024-11-16T00:00:55,811 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d31893a761fb01419b5d3147eeec1a/C in TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:55,811 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/80cc13e8bb4b44faa4912b4ccf2e737b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp, totalSize=49.0 K 2024-11-16T00:00:55,811 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cc13e8bb4b44faa4912b4ccf2e737b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731715252632 2024-11-16T00:00:55,812 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6b1d7a1a8647ad887451d7a4ee54c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731715253006 2024-11-16T00:00:55,812 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed2b75915024475a4569163381dc44b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731715253695 2024-11-16T00:00:55,812 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8122668980482f8272d2616e27fed5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731715254909 2024-11-16T00:00:55,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742399_1575 (size=13425) 2024-11-16T00:00:55,829 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d31893a761fb01419b5d3147eeec1a#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:00:55,829 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/99bd9e1c524442e3a30cbd6b7d017e38 is 50, key is test_row_0/C:col10/1731715254909/Put/seqid=0 2024-11-16T00:00:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T00:00:55,834 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/86b85faf03584c89a5d4248b1c043826 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/86b85faf03584c89a5d4248b1c043826 2024-11-16T00:00:55,843 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/A of e6d31893a761fb01419b5d3147eeec1a into 86b85faf03584c89a5d4248b1c043826(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:55,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:55,844 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/A, priority=12, startTime=1731715255750; duration=0sec 2024-11-16T00:00:55,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:55,844 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:55,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742400_1576 (size=13425) 2024-11-16T00:00:55,859 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/99bd9e1c524442e3a30cbd6b7d017e38 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/99bd9e1c524442e3a30cbd6b7d017e38 2024-11-16T00:00:55,867 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d31893a761fb01419b5d3147eeec1a/C of e6d31893a761fb01419b5d3147eeec1a into 99bd9e1c524442e3a30cbd6b7d017e38(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:00:55,867 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:55,867 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., storeName=e6d31893a761fb01419b5d3147eeec1a/C, priority=12, startTime=1731715255750; duration=0sec 2024-11-16T00:00:55,867 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:55,867 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:55,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-16T00:00:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:55,877 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:00:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/007301025a084726a3d7018c68515883 is 50, key is test_row_0/A:col10/1731715255282/Put/seqid=0 2024-11-16T00:00:55,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742401_1577 (size=12301) 2024-11-16T00:00:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:55,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715316016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715316018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715316020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715316021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715316021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T00:00:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715316140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715316140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715316140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715316140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715316140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,295 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/007301025a084726a3d7018c68515883 2024-11-16T00:00:56,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d1db1b17fb0b405aaf5e21b1f504e9e4 is 50, key is test_row_0/B:col10/1731715255282/Put/seqid=0 2024-11-16T00:00:56,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742402_1578 (size=12301) 2024-11-16T00:00:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T00:00:56,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715316359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715316359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715316360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715316360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715316360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:56,715 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d1db1b17fb0b405aaf5e21b1f504e9e4 2024-11-16T00:00:56,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/a28c60bea5f64a0584ded308a26a5a16 is 50, key is test_row_0/C:col10/1731715255282/Put/seqid=0 2024-11-16T00:00:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742403_1579 (size=12301) 2024-11-16T00:00:56,730 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/a28c60bea5f64a0584ded308a26a5a16 2024-11-16T00:00:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/007301025a084726a3d7018c68515883 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/007301025a084726a3d7018c68515883 2024-11-16T00:00:56,742 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/007301025a084726a3d7018c68515883, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T00:00:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/d1db1b17fb0b405aaf5e21b1f504e9e4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d1db1b17fb0b405aaf5e21b1f504e9e4 2024-11-16T00:00:56,748 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d1db1b17fb0b405aaf5e21b1f504e9e4, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T00:00:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/a28c60bea5f64a0584ded308a26a5a16 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a28c60bea5f64a0584ded308a26a5a16 2024-11-16T00:00:56,753 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a28c60bea5f64a0584ded308a26a5a16, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T00:00:56,754 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 877ms, sequenceid=490, compaction requested=false 2024-11-16T00:00:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-16T00:00:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-16T00:00:56,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T00:00:56,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-16T00:00:56,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.0400 sec 2024-11-16T00:00:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T00:00:56,860 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:56,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:00:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-16T00:00:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T00:00:56,866 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:00:56,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:00:56,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:00:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T00:00:57,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-16T00:00:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:57,021 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:00:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/e277ebaa0ef84213ad274291cc22b616 is 50, key is test_row_0/A:col10/1731715256018/Put/seqid=0 2024-11-16T00:00:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742404_1580 (size=12301) 2024-11-16T00:00:57,046 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/e277ebaa0ef84213ad274291cc22b616 2024-11-16T00:00:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/de566f8229b14adeb70e07cf95b721b5 is 50, key is test_row_0/B:col10/1731715256018/Put/seqid=0 2024-11-16T00:00:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742405_1581 (size=12301) 2024-11-16T00:00:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T00:00:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:57,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. as already flushing 2024-11-16T00:00:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715317216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715317218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715317219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715317220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715317229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715317340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715317340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715317340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715317340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715317350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,474 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/de566f8229b14adeb70e07cf95b721b5 2024-11-16T00:00:57,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1bc9948559294ff9901bdaea8ddf31e0 is 50, key is test_row_0/C:col10/1731715256018/Put/seqid=0 2024-11-16T00:00:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T00:00:57,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742406_1582 (size=12301) 2024-11-16T00:00:57,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715317560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715317560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715317560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715317560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715317570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52036 deadline: 1731715317880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52044 deadline: 1731715317880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52038 deadline: 1731715317880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52088 deadline: 1731715317880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,895 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1bc9948559294ff9901bdaea8ddf31e0 2024-11-16T00:00:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:00:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52086 deadline: 1731715317889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:00:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 , the old value is region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:00:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., hostname=0801813e30ae,43013,1731715110098, seqNum=2 because the exception is null or not the one we care about 2024-11-16T00:00:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/e277ebaa0ef84213ad274291cc22b616 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/e277ebaa0ef84213ad274291cc22b616 2024-11-16T00:00:57,918 INFO [Thread-2130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2130. 2024-11-16T00:00:57,918 DEBUG [Thread-2130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:57,918 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,919 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:57,923 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/e277ebaa0ef84213ad274291cc22b616, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T00:00:57,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/de566f8229b14adeb70e07cf95b721b5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/de566f8229b14adeb70e07cf95b721b5 2024-11-16T00:00:57,927 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-16T00:00:57,927 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:57,927 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,927 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:57,928 INFO [Thread-2128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2128. 2024-11-16T00:00:57,928 DEBUG [Thread-2128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:57,928 DEBUG [Thread-2128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:57,930 DEBUG [Thread-2128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,930 INFO [Thread-2126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2126. 2024-11-16T00:00:57,930 DEBUG [Thread-2126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:57,930 DEBUG [Thread-2126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,930 DEBUG [Thread-2126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:57,932 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/de566f8229b14adeb70e07cf95b721b5, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T00:00:57,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/1bc9948559294ff9901bdaea8ddf31e0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1bc9948559294ff9901bdaea8ddf31e0 2024-11-16T00:00:57,936 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-16T00:00:57,936 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:57,936 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,936 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:57,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:57,938 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1bc9948559294ff9901bdaea8ddf31e0, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T00:00:57,939 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e6d31893a761fb01419b5d3147eeec1a in 918ms, sequenceid=508, compaction requested=true 2024-11-16T00:00:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-16T00:00:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-16T00:00:57,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T00:00:57,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-16T00:00:57,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.0810 sec 2024-11-16T00:00:57,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T00:00:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T00:00:58,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:00:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:00:58,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:00:58,391 INFO [Thread-2121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2121. 2024-11-16T00:00:58,391 INFO [Thread-2117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2117. 2024-11-16T00:00:58,391 DEBUG [Thread-2121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:58,391 DEBUG [Thread-2117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:58,391 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,391 DEBUG [Thread-2117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,391 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,391 DEBUG [Thread-2117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/dffb901f381c44cab50a82eabdd07838 is 50, key is test_row_0/A:col10/1731715257214/Put/seqid=0 2024-11-16T00:00:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742407_1583 (size=12301) 2024-11-16T00:00:58,401 INFO [Thread-2123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2123. 2024-11-16T00:00:58,401 INFO [Thread-2115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2115. 2024-11-16T00:00:58,401 DEBUG [Thread-2123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:58,401 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,401 DEBUG [Thread-2115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:58,401 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,401 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,401 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,421 INFO [Thread-2119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2119. 2024-11-16T00:00:58,421 DEBUG [Thread-2119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:00:58,421 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,421 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1035 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3104 rows 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1018 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1025 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3073 rows 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1025 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3072 rows 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-11-16T00:00:58,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3070 rows 2024-11-16T00:00:58,422 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:00:58,422 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:00:58,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:00:58,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:00:58,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-16T00:00:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T00:00:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:00:58,426 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715258426"}]},"ts":"1731715258426"} 2024-11-16T00:00:58,427 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T00:00:58,427 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T00:00:58,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:00:58,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, UNASSIGN}] 2024-11-16T00:00:58,429 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, UNASSIGN 2024-11-16T00:00:58,430 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6d31893a761fb01419b5d3147eeec1a, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:00:58,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, UNASSIGN because future has completed 2024-11-16T00:00:58,431 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:00:58,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:00:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:00:58,584 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:00:58,584 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:00:58,584 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing e6d31893a761fb01419b5d3147eeec1a, disabling compactions & flushes 2024-11-16T00:00:58,584 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:00:58,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/dffb901f381c44cab50a82eabdd07838 2024-11-16T00:00:58,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/38f28ae5587a4ce6820bd384c0a55037 is 50, key is test_row_0/B:col10/1731715257214/Put/seqid=0 2024-11-16T00:00:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742408_1584 (size=12301) 2024-11-16T00:00:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:00:59,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/38f28ae5587a4ce6820bd384c0a55037 2024-11-16T00:00:59,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ceb27e71b6c24dd38083b2e415a7b462 is 50, key is test_row_0/C:col10/1731715257214/Put/seqid=0 2024-11-16T00:00:59,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742409_1585 (size=12301) 2024-11-16T00:00:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:00:59,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ceb27e71b6c24dd38083b2e415a7b462 2024-11-16T00:00:59,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/dffb901f381c44cab50a82eabdd07838 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/dffb901f381c44cab50a82eabdd07838 2024-11-16T00:00:59,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/dffb901f381c44cab50a82eabdd07838, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T00:00:59,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/38f28ae5587a4ce6820bd384c0a55037 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/38f28ae5587a4ce6820bd384c0a55037 2024-11-16T00:00:59,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/38f28ae5587a4ce6820bd384c0a55037, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T00:00:59,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/ceb27e71b6c24dd38083b2e415a7b462 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ceb27e71b6c24dd38083b2e415a7b462 2024-11-16T00:00:59,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ceb27e71b6c24dd38083b2e415a7b462, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T00:00:59,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for e6d31893a761fb01419b5d3147eeec1a in 1241ms, sequenceid=528, compaction requested=true 2024-11-16T00:00:59,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d31893a761fb01419b5d3147eeec1a: 2024-11-16T00:00:59,633 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:59,633 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. because compaction request was cancelled 2024-11-16T00:00:59,633 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. after waiting 0 ms 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:59,633 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:A 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. because compaction request was cancelled 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:B, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:B 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:59,633 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing e6d31893a761fb01419b5d3147eeec1a 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d31893a761fb01419b5d3147eeec1a:C, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. because compaction request was cancelled 2024-11-16T00:00:59,633 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d31893a761fb01419b5d3147eeec1a:C 2024-11-16T00:00:59,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:00:59,636 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/6a5579508d8743b4bf27971c603bc329 is 50, key is test_row_1/A:col10/1731715258420/Put/seqid=0 2024-11-16T00:00:59,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742410_1586 (size=7415) 2024-11-16T00:01:00,039 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/6a5579508d8743b4bf27971c603bc329 2024-11-16T00:01:00,046 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8e04a4608cc841a2b016f7df5ef9c828 is 50, key is test_row_1/B:col10/1731715258420/Put/seqid=0 2024-11-16T00:01:00,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742411_1587 (size=7415) 2024-11-16T00:01:00,052 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8e04a4608cc841a2b016f7df5ef9c828 2024-11-16T00:01:00,058 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3481a219f5bc4b00957474ccfe168eab is 50, key is test_row_1/C:col10/1731715258420/Put/seqid=0 2024-11-16T00:01:00,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742412_1588 (size=7415) 2024-11-16T00:01:00,063 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3481a219f5bc4b00957474ccfe168eab 2024-11-16T00:01:00,066 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/A/6a5579508d8743b4bf27971c603bc329 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6a5579508d8743b4bf27971c603bc329 2024-11-16T00:01:00,069 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6a5579508d8743b4bf27971c603bc329, entries=50, sequenceid=534, filesize=7.2 K 2024-11-16T00:01:00,070 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/B/8e04a4608cc841a2b016f7df5ef9c828 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8e04a4608cc841a2b016f7df5ef9c828 2024-11-16T00:01:00,074 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8e04a4608cc841a2b016f7df5ef9c828, entries=50, sequenceid=534, filesize=7.2 K 2024-11-16T00:01:00,074 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/.tmp/C/3481a219f5bc4b00957474ccfe168eab as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3481a219f5bc4b00957474ccfe168eab 2024-11-16T00:01:00,078 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3481a219f5bc4b00957474ccfe168eab, entries=50, sequenceid=534, filesize=7.2 K 2024-11-16T00:01:00,078 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e6d31893a761fb01419b5d3147eeec1a in 445ms, sequenceid=534, compaction requested=true 2024-11-16T00:01:00,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0ab6a1dc44b64347a65270653378be51, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6930ff874eab499898721b780299e6a7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/c7dfa7814a3e4939832fb71a5c05d84f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/bd8b1bf12906429dbd13757042e99071, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/29e8cef55cc440e5bb7cef6b78f7e689, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/06f50ba3049743dab02bc2220610e554, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/693833e9257f4de2ae971a110bfd54ed, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f1eac32e0970455b81735c1b76d8a08d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5838b8048c354e0c97523071f37fac70, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588] to archive 2024-11-16T00:01:00,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:00,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f56b3b11c89d461882cee6a08ab8abb2 2024-11-16T00:01:00,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0d8087bca775407aaae6536343474bf8 2024-11-16T00:01:00,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0ab6a1dc44b64347a65270653378be51 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0ab6a1dc44b64347a65270653378be51 2024-11-16T00:01:00,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/eb7e2a4f7f5145089be06348a6e848cc 2024-11-16T00:01:00,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ffecf38b629e4b008b918d84bedcb245 2024-11-16T00:01:00,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/9b26abc40c354c8bb725a8fd329b0f3f 2024-11-16T00:01:00,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6930ff874eab499898721b780299e6a7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6930ff874eab499898721b780299e6a7 2024-11-16T00:01:00,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a7d23b79a90c40f8a023f6f6b063667c 2024-11-16T00:01:00,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/537e3324931b41f8b941e5aa1cfbd472 2024-11-16T00:01:00,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/c7dfa7814a3e4939832fb71a5c05d84f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/c7dfa7814a3e4939832fb71a5c05d84f 2024-11-16T00:01:00,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4e9e3f46d23441198cb65707bbefc26b 2024-11-16T00:01:00,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/a65a5f29988e4f9cb8039d831f8e211e 2024-11-16T00:01:00,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/bd8b1bf12906429dbd13757042e99071 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/bd8b1bf12906429dbd13757042e99071 2024-11-16T00:01:00,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/73df2f06cc2c4e0882093135e99c3eba 2024-11-16T00:01:00,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/996bb9b689814cc0843d72fc74a7f71f 2024-11-16T00:01:00,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/29e8cef55cc440e5bb7cef6b78f7e689 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/29e8cef55cc440e5bb7cef6b78f7e689 2024-11-16T00:01:00,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/25848b44d9b14eadb3c14cf461ec40fd 2024-11-16T00:01:00,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/63bbec44e8aa4e46a9a2350d60b87aed 2024-11-16T00:01:00,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/06f50ba3049743dab02bc2220610e554 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/06f50ba3049743dab02bc2220610e554 2024-11-16T00:01:00,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/cbfff99e1d474d3a8197d3bdcbfd20e7 2024-11-16T00:01:00,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/693833e9257f4de2ae971a110bfd54ed to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/693833e9257f4de2ae971a110bfd54ed 2024-11-16T00:01:00,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/941decf7722742bab4bdb820e1aa0bf7 2024-11-16T00:01:00,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5b9fd7f1e5504e849eccdf854fb99d63 2024-11-16T00:01:00,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/8cdd5ba00ece4c72b80af77339affe1e 2024-11-16T00:01:00,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/20de24af87514e6d9db777f3ffdc36f6 2024-11-16T00:01:00,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f1eac32e0970455b81735c1b76d8a08d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/f1eac32e0970455b81735c1b76d8a08d 2024-11-16T00:01:00,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ad34d7d41d75407b87bc37b563355858 2024-11-16T00:01:00,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/0bb9276241c5498e839ebada45de8394 2024-11-16T00:01:00,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/85426e9e42644bfc8654e839dac0dafc 2024-11-16T00:01:00,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5838b8048c354e0c97523071f37fac70 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/5838b8048c354e0c97523071f37fac70 2024-11-16T00:01:00,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/1a7ad360d9ee4fd8ae26b5ca5f91768c 2024-11-16T00:01:00,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/4b000f02a21048e8925f8a161e618f2b 2024-11-16T00:01:00,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/ae8f786931c84ad18a7ce67cc7885588 2024-11-16T00:01:00,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/eed1c70a911949cd8e13823de612b415, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/2dd818dc305b4e52a5c14fe8380df026, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a0d89f04f4ac469db3491af07aa9e223, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a552d9be4b9140f98b8d9e832dca5ee9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/64a06ad131e14fa4a39510ebbf8a6861, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d924c03ceac1410a8f15c20acda876bf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a062011f7dca4c65beadf1443b3303e9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/241eb907f83f452c93a338af9212ba3e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/e1147271881c4684ac07586edc151ce3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22] to archive 2024-11-16T00:01:00,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:00,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8d0c90841c9e42ca9b924e31a604e63a 2024-11-16T00:01:00,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f4757903f844ca6aa0e5b38e973b770 2024-11-16T00:01:00,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/eed1c70a911949cd8e13823de612b415 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/eed1c70a911949cd8e13823de612b415 2024-11-16T00:01:00,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/54099feee58542518971a004baa2906d 2024-11-16T00:01:00,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc1d0dc2322d4ab0a168bc072850403b 2024-11-16T00:01:00,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/2dd818dc305b4e52a5c14fe8380df026 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/2dd818dc305b4e52a5c14fe8380df026 2024-11-16T00:01:00,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8f76e4afb7734b8c96a83d37a3e861fb 2024-11-16T00:01:00,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d97d3e6e79af4243b51d571d6a1cbb9a 2024-11-16T00:01:00,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a0d89f04f4ac469db3491af07aa9e223 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a0d89f04f4ac469db3491af07aa9e223 2024-11-16T00:01:00,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/fc2e502345714b5daa5a09fe211b67b5 2024-11-16T00:01:00,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/cb6bdcb996c6483a94c0988ff3fffb6b 2024-11-16T00:01:00,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a552d9be4b9140f98b8d9e832dca5ee9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a552d9be4b9140f98b8d9e832dca5ee9 2024-11-16T00:01:00,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/28a59f61633c469c803c61918c9f272e 2024-11-16T00:01:00,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/702d87d41b7249c093619ade5fa97ddb 2024-11-16T00:01:00,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/64a06ad131e14fa4a39510ebbf8a6861 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/64a06ad131e14fa4a39510ebbf8a6861 2024-11-16T00:01:00,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b145a367b9184449a1fccb7c84a91747 2024-11-16T00:01:00,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/321427ec0a744256861a48e40596f299 2024-11-16T00:01:00,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d924c03ceac1410a8f15c20acda876bf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d924c03ceac1410a8f15c20acda876bf 2024-11-16T00:01:00,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d4feeb9637254bd3b1d11e02b78323f0 2024-11-16T00:01:00,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/13a424ca27a540d1a135d5b6054a72e2 2024-11-16T00:01:00,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a062011f7dca4c65beadf1443b3303e9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/a062011f7dca4c65beadf1443b3303e9 2024-11-16T00:01:00,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/14fcca67a48d472c8b9653622da3a26e 2024-11-16T00:01:00,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/0c106d40c07c41d5806ac07db7678b21 2024-11-16T00:01:00,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/6564e5b4edf64f5f901f71e43d656268 2024-11-16T00:01:00,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/241eb907f83f452c93a338af9212ba3e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/241eb907f83f452c93a338af9212ba3e 2024-11-16T00:01:00,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/b8d47749ba45414cba96ae3651f79543 2024-11-16T00:01:00,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/5f5425cd8330410ea18ccd938609b6d1 2024-11-16T00:01:00,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/7674d6175e4f426c9fb63136f8e1c398 2024-11-16T00:01:00,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/e1147271881c4684ac07586edc151ce3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/e1147271881c4684ac07586edc151ce3 2024-11-16T00:01:00,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8a89cc0478424ab2a1ec2aeb4fdb85a5 2024-11-16T00:01:00,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/3788af4bd80a4b58b94bd0ea4c138c13 2024-11-16T00:01:00,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/c37badaa4fe54f61b24669811a4a2a67 2024-11-16T00:01:00,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/445725df1dd0440f978ce0c77097af22 2024-11-16T00:01:00,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/5af5b221d65b4156915c815bc37a8ebb, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a3db7d7bee1f47258f2b2730ad402d2d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bb7d1f0cb261469dbbed773ce3f8df07, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/adba1bc4ff9a47a4839929792e5bc60a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0b2120165d7b44a3b56369eb4be01d9d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/76084f767bd84bc29ab6dded27d88526, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/da988b551eb848f085be05b106593139, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/180c5d6677d64a18a96169414eee79db, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/80cc13e8bb4b44faa4912b4ccf2e737b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5] to archive 2024-11-16T00:01:00,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:00,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2f8a04728d7848598801da4b7063c428 2024-11-16T00:01:00,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/30ae12151ce94448a8a4cf67b06c9bfe 2024-11-16T00:01:00,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/5af5b221d65b4156915c815bc37a8ebb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/5af5b221d65b4156915c815bc37a8ebb 2024-11-16T00:01:00,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f499777a377c486e960114f779af6e06 2024-11-16T00:01:00,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d1c7924189f64945bf6c0e96f3279384 2024-11-16T00:01:00,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a3db7d7bee1f47258f2b2730ad402d2d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a3db7d7bee1f47258f2b2730ad402d2d 2024-11-16T00:01:00,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0c632d1b47ae4e06a86a01ec16097f5f 2024-11-16T00:01:00,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/037d4817a43a4d3e903350dca4eb5370 2024-11-16T00:01:00,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bb7d1f0cb261469dbbed773ce3f8df07 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bb7d1f0cb261469dbbed773ce3f8df07 2024-11-16T00:01:00,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/9fe2aabd0b9542f491ed9775c3218e29 2024-11-16T00:01:00,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/06fee4d1edb341b69119df72f717bf57 2024-11-16T00:01:00,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/adba1bc4ff9a47a4839929792e5bc60a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/adba1bc4ff9a47a4839929792e5bc60a 2024-11-16T00:01:00,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/c83a4a13510a4726b8ef5faad1274624 2024-11-16T00:01:00,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/7649f3898704409695fb587cee35eb75 2024-11-16T00:01:00,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0b2120165d7b44a3b56369eb4be01d9d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0b2120165d7b44a3b56369eb4be01d9d 2024-11-16T00:01:00,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ea0a67f156554dea85a92e41360614d0 2024-11-16T00:01:00,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/0afe87c2a9914a94b6b9cafe6228e52c 2024-11-16T00:01:00,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/76084f767bd84bc29ab6dded27d88526 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/76084f767bd84bc29ab6dded27d88526 2024-11-16T00:01:00,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/352d98be46aa41cab7e1c6a468be8e2b 2024-11-16T00:01:00,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/95955f4274954f1bac559d7aa0d3f471 2024-11-16T00:01:00,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/da988b551eb848f085be05b106593139 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/da988b551eb848f085be05b106593139 2024-11-16T00:01:00,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1dc3569bf46545df96e9616d641730de 2024-11-16T00:01:00,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/d7ccc543865840a29b28657b3d50704a 2024-11-16T00:01:00,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/bc9bd1be58f34af7ac00e555a6d87525 2024-11-16T00:01:00,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/180c5d6677d64a18a96169414eee79db to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/180c5d6677d64a18a96169414eee79db 2024-11-16T00:01:00,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/f08a08d454524ae380ccf48322196aa1 2024-11-16T00:01:00,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8e6d236556f44fc9905e9a62cf41ea42 2024-11-16T00:01:00,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/994ddc5cb03a474483ae955262a2511a 2024-11-16T00:01:00,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/80cc13e8bb4b44faa4912b4ccf2e737b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/80cc13e8bb4b44faa4912b4ccf2e737b 2024-11-16T00:01:00,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/8efe31e5f35e4497a79c58110673f117 2024-11-16T00:01:00,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3d6b1d7a1a8647ad887451d7a4ee54c9 2024-11-16T00:01:00,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3ed2b75915024475a4569163381dc44b 2024-11-16T00:01:00,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/2a8122668980482f8272d2616e27fed5 2024-11-16T00:01:00,211 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/recovered.edits/537.seqid, newMaxSeqId=537, maxSeqId=1 2024-11-16T00:01:00,211 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. 2024-11-16T00:01:00,212 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for e6d31893a761fb01419b5d3147eeec1a: Waiting for close lock at 1731715258584Running coprocessor pre-close hooks at 1731715258584Disabling compacts and flushes for region at 1731715258584Disabling writes for close at 1731715259633 (+1049 ms)Obtaining lock to block concurrent updates at 1731715259633Preparing flush snapshotting stores in e6d31893a761fb01419b5d3147eeec1a at 1731715259633Finished memstore snapshotting TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731715259633Flushing stores of TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a. at 1731715259633Flushing e6d31893a761fb01419b5d3147eeec1a/A: creating writer at 1731715259633Flushing e6d31893a761fb01419b5d3147eeec1a/A: appending metadata at 1731715259635 (+2 ms)Flushing e6d31893a761fb01419b5d3147eeec1a/A: closing flushed file at 1731715259635Flushing e6d31893a761fb01419b5d3147eeec1a/B: creating writer at 1731715260043 (+408 ms)Flushing e6d31893a761fb01419b5d3147eeec1a/B: appending metadata at 1731715260045 (+2 ms)Flushing e6d31893a761fb01419b5d3147eeec1a/B: closing flushed file at 1731715260045Flushing e6d31893a761fb01419b5d3147eeec1a/C: creating writer at 1731715260055 (+10 ms)Flushing e6d31893a761fb01419b5d3147eeec1a/C: appending metadata at 1731715260057 (+2 ms)Flushing e6d31893a761fb01419b5d3147eeec1a/C: closing flushed file at 1731715260057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bf52ef9: reopening flushed file at 1731715260066 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e13c63e: reopening flushed file at 1731715260070 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb0ea6c: reopening flushed file at 1731715260074 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e6d31893a761fb01419b5d3147eeec1a in 445ms, sequenceid=534, compaction requested=true at 1731715260079 (+5 ms)Writing region close event to WAL at 1731715260205 (+126 ms)Running coprocessor post-close hooks at 1731715260211 (+6 ms)Closed at 1731715260211 2024-11-16T00:01:00,214 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:01:00,215 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6d31893a761fb01419b5d3147eeec1a, regionState=CLOSED 2024-11-16T00:01:00,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:01:00,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T00:01:00,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure e6d31893a761fb01419b5d3147eeec1a, server=0801813e30ae,43013,1731715110098 in 1.7870 sec 2024-11-16T00:01:00,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T00:01:00,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d31893a761fb01419b5d3147eeec1a, UNASSIGN in 1.7910 sec 2024-11-16T00:01:00,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T00:01:00,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7960 sec 2024-11-16T00:01:00,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715260226"}]},"ts":"1731715260226"} 2024-11-16T00:01:00,227 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T00:01:00,227 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T00:01:00,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8050 sec 2024-11-16T00:01:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T00:01:00,570 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:00,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-16T00:01:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,572 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-16T00:01:00,572 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:01:00,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/recovered.edits] 2024-11-16T00:01:00,578 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/007301025a084726a3d7018c68515883 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/007301025a084726a3d7018c68515883 2024-11-16T00:01:00,579 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6a5579508d8743b4bf27971c603bc329 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/6a5579508d8743b4bf27971c603bc329 2024-11-16T00:01:00,580 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/86b85faf03584c89a5d4248b1c043826 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/86b85faf03584c89a5d4248b1c043826 2024-11-16T00:01:00,581 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/dffb901f381c44cab50a82eabdd07838 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/dffb901f381c44cab50a82eabdd07838 2024-11-16T00:01:00,582 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/e277ebaa0ef84213ad274291cc22b616 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/A/e277ebaa0ef84213ad274291cc22b616 2024-11-16T00:01:00,584 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/38f28ae5587a4ce6820bd384c0a55037 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/38f28ae5587a4ce6820bd384c0a55037 2024-11-16T00:01:00,585 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8e04a4608cc841a2b016f7df5ef9c828 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/8e04a4608cc841a2b016f7df5ef9c828 2024-11-16T00:01:00,586 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d1db1b17fb0b405aaf5e21b1f504e9e4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/d1db1b17fb0b405aaf5e21b1f504e9e4 2024-11-16T00:01:00,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/de566f8229b14adeb70e07cf95b721b5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/de566f8229b14adeb70e07cf95b721b5 2024-11-16T00:01:00,588 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/ee40bc54827c403db401fb80bc30f73c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/B/ee40bc54827c403db401fb80bc30f73c 2024-11-16T00:01:00,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1bc9948559294ff9901bdaea8ddf31e0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/1bc9948559294ff9901bdaea8ddf31e0 2024-11-16T00:01:00,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3481a219f5bc4b00957474ccfe168eab to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/3481a219f5bc4b00957474ccfe168eab 2024-11-16T00:01:00,592 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/99bd9e1c524442e3a30cbd6b7d017e38 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/99bd9e1c524442e3a30cbd6b7d017e38 2024-11-16T00:01:00,592 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a28c60bea5f64a0584ded308a26a5a16 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/a28c60bea5f64a0584ded308a26a5a16 2024-11-16T00:01:00,593 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ceb27e71b6c24dd38083b2e415a7b462 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/C/ceb27e71b6c24dd38083b2e415a7b462 2024-11-16T00:01:00,595 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/recovered.edits/537.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a/recovered.edits/537.seqid 2024-11-16T00:01:00,595 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/e6d31893a761fb01419b5d3147eeec1a 2024-11-16T00:01:00,596 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T00:01:00,598 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,600 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T00:01:00,602 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T00:01:00,604 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T00:01:00,605 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715260605"}]},"ts":"9223372036854775807"} 2024-11-16T00:01:00,607 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T00:01:00,607 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e6d31893a761fb01419b5d3147eeec1a, NAME => 'TestAcidGuarantees,,1731715235725.e6d31893a761fb01419b5d3147eeec1a.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T00:01:00,607 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T00:01:00,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715260607"}]},"ts":"9223372036854775807"} 2024-11-16T00:01:00,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T00:01:00,609 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-16T00:01:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-16T00:01:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T00:01:00,682 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:00,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 241), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=666 (was 651) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4469 (was 5003) 2024-11-16T00:01:00,701 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=11, AvailableMemoryMB=4468 2024-11-16T00:01:00,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T00:01:00,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T00:01:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:00,704 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T00:01:00,705 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:00,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-16T00:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:00,706 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T00:01:00,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742413_1589 (size=959) 2024-11-16T00:01:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:01,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e 2024-11-16T00:01:01,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742414_1590 (size=53) 2024-11-16T00:01:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 895ec0f83f32ef97e348b295ffa56e5a, disabling compactions & flushes 2024-11-16T00:01:01,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. after waiting 0 ms 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 895ec0f83f32ef97e348b295ffa56e5a: Waiting for close lock at 1731715261520Disabling compacts and flushes for region at 1731715261520Disabling writes for close at 1731715261520Writing region close event to WAL at 1731715261520Closed at 1731715261520 2024-11-16T00:01:01,521 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T00:01:01,522 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731715261521"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731715261521"}]},"ts":"1731715261521"} 2024-11-16T00:01:01,523 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T00:01:01,524 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T00:01:01,524 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715261524"}]},"ts":"1731715261524"} 2024-11-16T00:01:01,525 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T00:01:01,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, ASSIGN}] 2024-11-16T00:01:01,526 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, ASSIGN 2024-11-16T00:01:01,526 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, ASSIGN; state=OFFLINE, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=false 2024-11-16T00:01:01,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:01,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, ASSIGN because future has completed 2024-11-16T00:01:01,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:01:01,834 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,835 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T00:01:01,835 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,835 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:01:01,835 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,835 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,836 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,839 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName A 2024-11-16T00:01:01,839 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:01,839 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:01,840 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,841 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName B 2024-11-16T00:01:01,841 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:01,841 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:01,841 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,844 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName C 2024-11-16T00:01:01,844 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:01,844 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:01,844 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,845 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,846 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,847 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,847 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,847 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T00:01:01,848 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,851 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T00:01:01,852 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 895ec0f83f32ef97e348b295ffa56e5a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67741225, jitterRate=0.009422913193702698}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T00:01:01,852 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:01,853 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 895ec0f83f32ef97e348b295ffa56e5a: Running coprocessor pre-open hook at 1731715261835Writing region info on filesystem at 1731715261835Initializing all the Stores at 1731715261836 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715261836Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715261836Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715261836Cleaning up temporary data from old regions at 1731715261847 (+11 ms)Running coprocessor post-open hooks at 1731715261852 (+5 ms)Region opened successfully at 1731715261853 (+1 ms) 2024-11-16T00:01:01,854 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., pid=152, masterSystemTime=1731715261831 2024-11-16T00:01:01,856 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,856 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:01,856 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=OPEN, openSeqNum=2, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:01,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:01:01,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-16T00:01:01,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 in 180 msec 2024-11-16T00:01:01,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T00:01:01,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, ASSIGN in 335 msec 2024-11-16T00:01:01,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T00:01:01,863 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715261863"}]},"ts":"1731715261863"} 2024-11-16T00:01:01,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T00:01:01,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T00:01:01,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-16T00:01:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T00:01:02,850 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:02,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:02,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:02,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:02,852 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a001436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:02,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4266a3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:02,855 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:02,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:02,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T00:01:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 0801813e30ae,38967,1731715109050 2024-11-16T00:01:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@155cf507 2024-11-16T00:01:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T00:01:02,859 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T00:01:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T00:01:02,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T00:01:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:02,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742415_1591 (size=995) 2024-11-16T00:01:03,273 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T00:01:03,273 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T00:01:03,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:01:03,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE}] 2024-11-16T00:01:03,278 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE 2024-11-16T00:01:03,279 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE because future has completed 2024-11-16T00:01:03,280 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:01:03,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:01:03,432 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,432 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:01:03,432 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 895ec0f83f32ef97e348b295ffa56e5a, disabling compactions & flushes 2024-11-16T00:01:03,432 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,433 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,433 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. after waiting 0 ms 2024-11-16T00:01:03,433 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,436 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T00:01:03,436 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,436 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 895ec0f83f32ef97e348b295ffa56e5a: Waiting for close lock at 1731715263432Running coprocessor pre-close hooks at 1731715263432Disabling compacts and flushes for region at 1731715263432Disabling writes for close at 1731715263433 (+1 ms)Writing region close event to WAL at 1731715263433Running coprocessor post-close hooks at 1731715263436 (+3 ms)Closed at 1731715263436 2024-11-16T00:01:03,436 WARN [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 895ec0f83f32ef97e348b295ffa56e5a to self. 2024-11-16T00:01:03,438 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,438 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=CLOSED 2024-11-16T00:01:03,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:01:03,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-16T00:01:03,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 in 160 msec 2024-11-16T00:01:03,443 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE; state=CLOSED, location=0801813e30ae,43013,1731715110098; forceNewPlan=false, retain=true 2024-11-16T00:01:03,593 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=OPENING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE because future has completed 2024-11-16T00:01:03,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:01:03,749 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,749 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T00:01:03,750 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,750 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T00:01:03,750 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,750 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,751 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,751 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName A 2024-11-16T00:01:03,753 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:03,753 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:03,753 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,754 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName B 2024-11-16T00:01:03,754 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:03,755 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:03,755 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,755 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895ec0f83f32ef97e348b295ffa56e5a columnFamilyName C 2024-11-16T00:01:03,755 DEBUG [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:03,756 INFO [StoreOpener-895ec0f83f32ef97e348b295ffa56e5a-1 {}] regionserver.HStore(327): Store=895ec0f83f32ef97e348b295ffa56e5a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T00:01:03,756 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,756 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,757 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,758 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,758 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,758 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T00:01:03,759 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,760 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 895ec0f83f32ef97e348b295ffa56e5a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64074847, jitterRate=-0.045210376381874084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T00:01:03,760 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,760 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 895ec0f83f32ef97e348b295ffa56e5a: Running coprocessor pre-open hook at 1731715263750Writing region info on filesystem at 1731715263750Initializing all the Stores at 1731715263750Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715263750Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715263751 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731715263751Cleaning up temporary data from old regions at 1731715263758 (+7 ms)Running coprocessor post-open hooks at 1731715263760 (+2 ms)Region opened successfully at 1731715263760 2024-11-16T00:01:03,761 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., pid=157, masterSystemTime=1731715263746 2024-11-16T00:01:03,763 DEBUG [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,763 INFO [RS_OPEN_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,763 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=OPEN, openSeqNum=5, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:01:03,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-16T00:01:03,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 in 170 msec 2024-11-16T00:01:03,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T00:01:03,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, REOPEN/MOVE in 489 msec 2024-11-16T00:01:03,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-16T00:01:03,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-16T00:01:03,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-16T00:01:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-16T00:01:03,774 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:03,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,776 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42eb8894, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,777 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573a3a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f30b1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769bb594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0198c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf0063a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d16522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa4d34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,787 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c7b5b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9edc6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,790 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424fb8f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,793 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7851b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,794 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f0757a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,796 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3623096f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,797 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69785ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,799 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9814b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,800 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167e1f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742259fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0801813e30ae,38967,-1 for getting cluster id 2024-11-16T00:01:03,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T00:01:03,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '774650c6-d881-48c6-afec-cf67c8e46ad4' 2024-11-16T00:01:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T00:01:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "774650c6-d881-48c6-afec-cf67c8e46ad4" 2024-11-16T00:01:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa00ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0801813e30ae,38967,-1] 2024-11-16T00:01:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T00:01:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:03,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T00:01:03,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d95496b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T00:01:03,803 DEBUG [Thread-2619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,803 DEBUG [Thread-2615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,803 DEBUG [Thread-2622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,803 DEBUG [Thread-2628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,804 DEBUG [Thread-2613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,804 DEBUG [Thread-2624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,804 DEBUG [Thread-2617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:03,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T00:01:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T00:01:03,811 DEBUG [Thread-2626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,813 DEBUG [Thread-2630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,814 DEBUG [Thread-2611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T00:01:03,815 DEBUG [Thread-2611 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,815 DEBUG [Thread-2611 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0801813e30ae,43013,1731715110098, seqNum=-1] 2024-11-16T00:01:03,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T00:01:03,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T00:01:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5] 2024-11-16T00:01:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,822 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:03,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T00:01:03,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715323851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715323852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715323853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715323853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715323856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116025dc14ddabb4e849df5366cc6942283_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715263821/Put/seqid=0 2024-11-16T00:01:03,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742416_1592 (size=14594) 2024-11-16T00:01:03,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:03,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116025dc14ddabb4e849df5366cc6942283_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116025dc14ddabb4e849df5366cc6942283_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:03,881 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7142e6b8af4c40e1ad8b4fd0b2a52485, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:03,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7142e6b8af4c40e1ad8b4fd0b2a52485 is 175, key is test_row_0/A:col10/1731715263821/Put/seqid=0 2024-11-16T00:01:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T00:01:03,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742417_1593 (size=39549) 2024-11-16T00:01:03,932 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7142e6b8af4c40e1ad8b4fd0b2a52485 2024-11-16T00:01:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715323960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715323960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715323960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715323960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715323969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:03,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T00:01:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:03,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:03,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:03,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b52909e142c49a5b93f35f68d669a85 is 50, key is test_row_0/B:col10/1731715263821/Put/seqid=0 2024-11-16T00:01:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742418_1594 (size=12001) 2024-11-16T00:01:04,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b52909e142c49a5b93f35f68d669a85 2024-11-16T00:01:04,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/45d8d2fce5dd4e939d2db121c957b46b is 50, key is test_row_0/C:col10/1731715263821/Put/seqid=0 2024-11-16T00:01:04,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742419_1595 (size=12001) 2024-11-16T00:01:04,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/45d8d2fce5dd4e939d2db121c957b46b 2024-11-16T00:01:04,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7142e6b8af4c40e1ad8b4fd0b2a52485 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485 2024-11-16T00:01:04,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485, entries=200, sequenceid=16, filesize=38.6 K 2024-11-16T00:01:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b52909e142c49a5b93f35f68d669a85 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85 2024-11-16T00:01:04,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T00:01:04,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/45d8d2fce5dd4e939d2db121c957b46b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b 2024-11-16T00:01:04,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T00:01:04,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 895ec0f83f32ef97e348b295ffa56e5a in 288ms, sequenceid=16, compaction requested=false 2024-11-16T00:01:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:04,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T00:01:04,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:04,125 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T00:01:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T00:01:04,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165ec940bbce964f7ca9ceb3cfb501fee4_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715263850/Put/seqid=0 2024-11-16T00:01:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742420_1596 (size=12154) 2024-11-16T00:01:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:04,165 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165ec940bbce964f7ca9ceb3cfb501fee4_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec940bbce964f7ca9ceb3cfb501fee4_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d16fb81e75af4ac1be536759ff3fb7c5, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:04,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d16fb81e75af4ac1be536759ff3fb7c5 is 175, key is test_row_0/A:col10/1731715263850/Put/seqid=0 2024-11-16T00:01:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:04,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742421_1597 (size=30955) 2024-11-16T00:01:04,179 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d16fb81e75af4ac1be536759ff3fb7c5 2024-11-16T00:01:04,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715324184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715324185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715324185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715324185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715324186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/03aefa808fa44e18a19f2b6333a7ed67 is 50, key is test_row_0/B:col10/1731715263850/Put/seqid=0 2024-11-16T00:01:04,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742422_1598 (size=12001) 2024-11-16T00:01:04,224 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/03aefa808fa44e18a19f2b6333a7ed67 2024-11-16T00:01:04,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/50b26a7f4f2c48e1b4fe5289e65cd488 is 50, key is test_row_0/C:col10/1731715263850/Put/seqid=0 2024-11-16T00:01:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742423_1599 (size=12001) 2024-11-16T00:01:04,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715324299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715324300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715324300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715324300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715324300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T00:01:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715324510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715324510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715324511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715324511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715324511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,639 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/50b26a7f4f2c48e1b4fe5289e65cd488 2024-11-16T00:01:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d16fb81e75af4ac1be536759ff3fb7c5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5 2024-11-16T00:01:04,655 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T00:01:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/03aefa808fa44e18a19f2b6333a7ed67 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67 2024-11-16T00:01:04,659 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T00:01:04,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/50b26a7f4f2c48e1b4fe5289e65cd488 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488 2024-11-16T00:01:04,663 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T00:01:04,664 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 895ec0f83f32ef97e348b295ffa56e5a in 539ms, sequenceid=40, compaction requested=false 2024-11-16T00:01:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T00:01:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T00:01:04,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T00:01:04,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 852 msec 2024-11-16T00:01:04,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 860 msec 2024-11-16T00:01:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:04,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:04,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T00:01:04,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161297647efa8744ba8d2cd69bab8c5e87_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:04,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715324866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715324867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715324868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715324870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742424_1600 (size=14594) 2024-11-16T00:01:04,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:04,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161297647efa8744ba8d2cd69bab8c5e87_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161297647efa8744ba8d2cd69bab8c5e87_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:04,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0e8e4d6e08f418c93b080d0ac58ba4b, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:04,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0e8e4d6e08f418c93b080d0ac58ba4b is 175, key is test_row_0/A:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:04,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742425_1601 (size=39549) 2024-11-16T00:01:04,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0e8e4d6e08f418c93b080d0ac58ba4b 2024-11-16T00:01:04,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/cea692b80ee14126b20311778f7f8200 is 50, key is test_row_0/B:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742426_1602 (size=12001) 2024-11-16T00:01:04,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/cea692b80ee14126b20311778f7f8200 2024-11-16T00:01:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T00:01:04,950 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:04,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T00:01:04,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T00:01:04,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:04,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:04,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/85db7999fc1b4a49879388aa9714e198 is 50, key is test_row_0/C:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:04,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715324980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715324980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715324980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715324980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715324980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742427_1603 (size=12001) 2024-11-16T00:01:05,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/85db7999fc1b4a49879388aa9714e198 2024-11-16T00:01:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0e8e4d6e08f418c93b080d0ac58ba4b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b 2024-11-16T00:01:05,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b, entries=200, sequenceid=56, filesize=38.6 K 2024-11-16T00:01:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/cea692b80ee14126b20311778f7f8200 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200 2024-11-16T00:01:05,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200, entries=150, sequenceid=56, filesize=11.7 K 2024-11-16T00:01:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T00:01:05,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/85db7999fc1b4a49879388aa9714e198 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198 2024-11-16T00:01:05,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198, entries=150, sequenceid=56, filesize=11.7 K 2024-11-16T00:01:05,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 895ec0f83f32ef97e348b295ffa56e5a in 244ms, sequenceid=56, compaction requested=true 2024-11-16T00:01:05,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:05,077 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:05,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:05,077 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:05,079 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:05,080 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:05,080 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=107.5 K 2024-11-16T00:01:05,080 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:05,080 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b] 2024-11-16T00:01:05,080 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=35.2 K 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7142e6b8af4c40e1ad8b4fd0b2a52485, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715263816 2024-11-16T00:01:05,080 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b52909e142c49a5b93f35f68d669a85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715263816 2024-11-16T00:01:05,081 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16fb81e75af4ac1be536759ff3fb7c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715263850 2024-11-16T00:01:05,081 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 03aefa808fa44e18a19f2b6333a7ed67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715263850 2024-11-16T00:01:05,081 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e8e4d6e08f418c93b080d0ac58ba4b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:05,081 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting cea692b80ee14126b20311778f7f8200, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:05,091 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:05,091 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4554718f914845988c68c1ec8c8eba9c is 50, key is test_row_0/B:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:05,094 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:05,096 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116608d2368a47049b3bd1d79c52a7cc9a1_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:05,097 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116608d2368a47049b3bd1d79c52a7cc9a1_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:05,097 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116608d2368a47049b3bd1d79c52a7cc9a1_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:05,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T00:01:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:05,115 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:01:05,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742428_1604 (size=12104) 2024-11-16T00:01:05,143 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4554718f914845988c68c1ec8c8eba9c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4554718f914845988c68c1ec8c8eba9c 2024-11-16T00:01:05,149 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 4554718f914845988c68c1ec8c8eba9c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:05,149 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:05,149 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=13, startTime=1731715265077; duration=0sec 2024-11-16T00:01:05,149 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:05,149 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:05,149 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:05,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742429_1605 (size=4469) 2024-11-16T00:01:05,153 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:05,153 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:05,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602e8d5a45c20473da5b41d699a19561f_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715264867/Put/seqid=0 2024-11-16T00:01:05,154 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:05,154 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=35.2 K 2024-11-16T00:01:05,155 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#514 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:05,155 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6b117e4829b94a2dbacfb8836c51293d is 175, key is test_row_0/A:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:05,156 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d8d2fce5dd4e939d2db121c957b46b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731715263816 2024-11-16T00:01:05,156 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b26a7f4f2c48e1b4fe5289e65cd488, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731715263850 2024-11-16T00:01:05,156 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 85db7999fc1b4a49879388aa9714e198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:05,179 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:05,180 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/d57ad5141a714e70a96f78bb70986290 is 50, key is test_row_0/C:col10/1731715264832/Put/seqid=0 2024-11-16T00:01:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742430_1606 (size=12154) 2024-11-16T00:01:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742431_1607 (size=31058) 2024-11-16T00:01:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:05,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742432_1608 (size=12104) 2024-11-16T00:01:05,199 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/d57ad5141a714e70a96f78bb70986290 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d57ad5141a714e70a96f78bb70986290 2024-11-16T00:01:05,207 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into d57ad5141a714e70a96f78bb70986290(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:05,207 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:05,207 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=13, startTime=1731715265077; duration=0sec 2024-11-16T00:01:05,207 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:05,207 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715325215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715325217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715325217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715325219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715325219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T00:01:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715325319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715325330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715325330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715325330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715325333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715325529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715325540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715325540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715325541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715325550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:05,587 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602e8d5a45c20473da5b41d699a19561f_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602e8d5a45c20473da5b41d699a19561f_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/9f5cf49fdd8847489b837805baebb48d, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/9f5cf49fdd8847489b837805baebb48d is 175, key is test_row_0/A:col10/1731715264867/Put/seqid=0 2024-11-16T00:01:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T00:01:05,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742433_1609 (size=30955) 2024-11-16T00:01:05,593 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6b117e4829b94a2dbacfb8836c51293d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d 2024-11-16T00:01:05,594 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/9f5cf49fdd8847489b837805baebb48d 2024-11-16T00:01:05,605 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into 6b117e4829b94a2dbacfb8836c51293d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:05,605 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:05,605 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=13, startTime=1731715265077; duration=0sec 2024-11-16T00:01:05,605 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:05,605 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ca95584297d44cadac51e0a481551fc3 is 50, key is test_row_0/B:col10/1731715264867/Put/seqid=0 2024-11-16T00:01:05,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742434_1610 (size=12001) 2024-11-16T00:01:05,737 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T00:01:05,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715325840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715325850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715325850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715325859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715325859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:05,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,013 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ca95584297d44cadac51e0a481551fc3 2024-11-16T00:01:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/a2ba0d3bdcf6447faeaf00588c608f41 is 50, key is test_row_0/C:col10/1731715264867/Put/seqid=0 2024-11-16T00:01:06,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742435_1611 (size=12001) 2024-11-16T00:01:06,028 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/a2ba0d3bdcf6447faeaf00588c608f41 2024-11-16T00:01:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/9f5cf49fdd8847489b837805baebb48d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d 2024-11-16T00:01:06,036 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-16T00:01:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ca95584297d44cadac51e0a481551fc3 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3 2024-11-16T00:01:06,040 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T00:01:06,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/a2ba0d3bdcf6447faeaf00588c608f41 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41 2024-11-16T00:01:06,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T00:01:06,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 895ec0f83f32ef97e348b295ffa56e5a in 931ms, sequenceid=76, compaction requested=false 2024-11-16T00:01:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T00:01:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T00:01:06,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T00:01:06,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-16T00:01:06,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.0920 sec 2024-11-16T00:01:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T00:01:06,100 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:06,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T00:01:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:06,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:06,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:06,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:06,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T00:01:06,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:06,257 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:01:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b075791a0e9643eeb39a78214f3206cc_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715265216/Put/seqid=0 2024-11-16T00:01:06,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742436_1612 (size=12154) 2024-11-16T00:01:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:06,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:06,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715326379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715326381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715326384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715326385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715326386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:06,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715326490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715326490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715326500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715326500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715326500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:06,677 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b075791a0e9643eeb39a78214f3206cc_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b075791a0e9643eeb39a78214f3206cc_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:06,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4b9aac03a3ab44a99157875b1243732c, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4b9aac03a3ab44a99157875b1243732c is 175, key is test_row_0/A:col10/1731715265216/Put/seqid=0 2024-11-16T00:01:06,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742437_1613 (size=30955) 2024-11-16T00:01:06,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715326700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715326700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715326709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715326710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715326710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715327009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715327009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715327019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715327020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715327020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,083 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4b9aac03a3ab44a99157875b1243732c 2024-11-16T00:01:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4a95d8cb79f24c5bbef803bc02894dbe is 50, key is test_row_0/B:col10/1731715265216/Put/seqid=0 2024-11-16T00:01:07,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742438_1614 (size=12001) 2024-11-16T00:01:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:07,497 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4a95d8cb79f24c5bbef803bc02894dbe 2024-11-16T00:01:07,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f89bab52157e4b4eb551a538d9eda226 is 50, key is test_row_0/C:col10/1731715265216/Put/seqid=0 2024-11-16T00:01:07,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715327521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715327521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715327530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715327530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715327531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:07,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742439_1615 (size=12001) 2024-11-16T00:01:07,935 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f89bab52157e4b4eb551a538d9eda226 2024-11-16T00:01:07,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4b9aac03a3ab44a99157875b1243732c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c 2024-11-16T00:01:07,946 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c, entries=150, sequenceid=96, filesize=30.2 K 2024-11-16T00:01:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4a95d8cb79f24c5bbef803bc02894dbe as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe 2024-11-16T00:01:07,950 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T00:01:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f89bab52157e4b4eb551a538d9eda226 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226 2024-11-16T00:01:07,955 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T00:01:07,956 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 895ec0f83f32ef97e348b295ffa56e5a in 1699ms, sequenceid=96, compaction requested=true 2024-11-16T00:01:07,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:07,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:07,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T00:01:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T00:01:07,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T00:01:07,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-16T00:01:07,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8590 sec 2024-11-16T00:01:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T00:01:08,250 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:08,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T00:01:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T00:01:08,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:08,253 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:08,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T00:01:08,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T00:01:08,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:08,406 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:01:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f496279f27d41a8a3acdc261d385739_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715266382/Put/seqid=0 2024-11-16T00:01:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742440_1616 (size=12154) 2024-11-16T00:01:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:08,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715328552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715328555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715328557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715328559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715328560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T00:01:08,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715328669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715328669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715328669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715328670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715328670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:08,831 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f496279f27d41a8a3acdc261d385739_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f496279f27d41a8a3acdc261d385739_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/bc9809ac0b314e299dab519d271d0cf1, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/bc9809ac0b314e299dab519d271d0cf1 is 175, key is test_row_0/A:col10/1731715266382/Put/seqid=0 2024-11-16T00:01:08,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742441_1617 (size=30955) 2024-11-16T00:01:08,837 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/bc9809ac0b314e299dab519d271d0cf1 2024-11-16T00:01:08,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/09e5a38d3ece422ca5fd97b9b2923578 is 50, key is test_row_0/B:col10/1731715266382/Put/seqid=0 2024-11-16T00:01:08,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742442_1618 (size=12001) 2024-11-16T00:01:08,850 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/09e5a38d3ece422ca5fd97b9b2923578 2024-11-16T00:01:08,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/cc3c94b0ae2241beadbad6e3125ed819 is 50, key is test_row_0/C:col10/1731715266382/Put/seqid=0 2024-11-16T00:01:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742443_1619 (size=12001) 2024-11-16T00:01:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T00:01:08,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715328881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715328881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715328881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715328881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715328881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,271 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/cc3c94b0ae2241beadbad6e3125ed819 2024-11-16T00:01:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/bc9809ac0b314e299dab519d271d0cf1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1 2024-11-16T00:01:09,278 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1, entries=150, sequenceid=115, filesize=30.2 K 2024-11-16T00:01:09,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/09e5a38d3ece422ca5fd97b9b2923578 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578 2024-11-16T00:01:09,282 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T00:01:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/cc3c94b0ae2241beadbad6e3125ed819 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819 2024-11-16T00:01:09,286 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T00:01:09,287 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 895ec0f83f32ef97e348b295ffa56e5a in 881ms, sequenceid=115, compaction requested=true 2024-11-16T00:01:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T00:01:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T00:01:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T00:01:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-11-16T00:01:09,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0420 sec 2024-11-16T00:01:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T00:01:09,390 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:09,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-16T00:01:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T00:01:09,393 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:09,393 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:09,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T00:01:09,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T00:01:09,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:09,546 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:01:09,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168f1ec153dcc245a8ad7e6d7b6dcdd68f_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715268555/Put/seqid=0 2024-11-16T00:01:09,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742444_1620 (size=12304) 2024-11-16T00:01:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:09,577 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168f1ec153dcc245a8ad7e6d7b6dcdd68f_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f1ec153dcc245a8ad7e6d7b6dcdd68f_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:09,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/cbcab3aee0e743f0ad4399745bcc595a, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/cbcab3aee0e743f0ad4399745bcc595a is 175, key is test_row_0/A:col10/1731715268555/Put/seqid=0 2024-11-16T00:01:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742445_1621 (size=31105) 2024-11-16T00:01:09,598 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/cbcab3aee0e743f0ad4399745bcc595a 2024-11-16T00:01:09,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/a4b1d736a97144699548dfe8385c5bcc is 50, key is test_row_0/B:col10/1731715268555/Put/seqid=0 2024-11-16T00:01:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742446_1622 (size=12151) 2024-11-16T00:01:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T00:01:09,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715329723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715329723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715329726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715329729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715329840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715329840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715329840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715329840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715329841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T00:01:10,020 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/a4b1d736a97144699548dfe8385c5bcc 2024-11-16T00:01:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/4d3b7179072647e2932244f1c65e3996 is 50, key is test_row_0/C:col10/1731715268555/Put/seqid=0 2024-11-16T00:01:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742447_1623 (size=12151) 2024-11-16T00:01:10,036 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/4d3b7179072647e2932244f1c65e3996 2024-11-16T00:01:10,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/cbcab3aee0e743f0ad4399745bcc595a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a 2024-11-16T00:01:10,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a, entries=150, sequenceid=132, filesize=30.4 K 2024-11-16T00:01:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/a4b1d736a97144699548dfe8385c5bcc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc 2024-11-16T00:01:10,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715330050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715330050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715330050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715330050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715330051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,060 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T00:01:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/4d3b7179072647e2932244f1c65e3996 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996 2024-11-16T00:01:10,078 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T00:01:10,078 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 895ec0f83f32ef97e348b295ffa56e5a in 532ms, sequenceid=132, compaction requested=true 2024-11-16T00:01:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-16T00:01:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-16T00:01:10,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T00:01:10,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 686 msec 2024-11-16T00:01:10,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 692 msec 2024-11-16T00:01:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:10,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:10,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:01:10,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c64e07bbc5924280917edbf9cc7e04ab_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742448_1624 (size=12304) 2024-11-16T00:01:10,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:10,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c64e07bbc5924280917edbf9cc7e04ab_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c64e07bbc5924280917edbf9cc7e04ab_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:10,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4bd6f956492b41c69a49328ffb49dbab, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:10,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4bd6f956492b41c69a49328ffb49dbab is 175, key is test_row_0/A:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715330385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715330385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715330385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715330387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715330390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742449_1625 (size=31105) 2024-11-16T00:01:10,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4bd6f956492b41c69a49328ffb49dbab 2024-11-16T00:01:10,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/5486427da0364db1ac1b92d7c4579ce6 is 50, key is test_row_0/B:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742450_1626 (size=12151) 2024-11-16T00:01:10,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/5486427da0364db1ac1b92d7c4579ce6 2024-11-16T00:01:10,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/13b9e2f3627b45299ef0d3c9c18a39db is 50, key is test_row_0/C:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742451_1627 (size=12151) 2024-11-16T00:01:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715330500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715330500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715330500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715330500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715330500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T00:01:10,530 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:10,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-16T00:01:10,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:10,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:10,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:10,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T00:01:10,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715330710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715330710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715330710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715330710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715330721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:10,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T00:01:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/13b9e2f3627b45299ef0d3c9c18a39db 2024-11-16T00:01:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:10,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/4bd6f956492b41c69a49328ffb49dbab as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab 2024-11-16T00:01:10,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab, entries=150, sequenceid=152, filesize=30.4 K 2024-11-16T00:01:10,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/5486427da0364db1ac1b92d7c4579ce6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6 2024-11-16T00:01:10,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6, entries=150, sequenceid=152, filesize=11.9 K 2024-11-16T00:01:10,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/13b9e2f3627b45299ef0d3c9c18a39db as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db 2024-11-16T00:01:10,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db, entries=150, sequenceid=152, filesize=11.9 K 2024-11-16T00:01:10,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 895ec0f83f32ef97e348b295ffa56e5a in 504ms, sequenceid=152, compaction requested=true 2024-11-16T00:01:10,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:10,867 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:10,868 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:10,869 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 186133 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T00:01:10,869 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:10,869 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,869 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=181.8 K 2024-11-16T00:01:10,869 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,869 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab] 2024-11-16T00:01:10,870 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b117e4829b94a2dbacfb8836c51293d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:10,872 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5cf49fdd8847489b837805baebb48d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715264867 2024-11-16T00:01:10,872 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9aac03a3ab44a99157875b1243732c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731715265209 2024-11-16T00:01:10,872 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72409 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T00:01:10,872 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:10,872 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:10,872 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4554718f914845988c68c1ec8c8eba9c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=70.7 K 2024-11-16T00:01:10,873 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4554718f914845988c68c1ec8c8eba9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:10,874 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9809ac0b314e299dab519d271d0cf1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731715266379 2024-11-16T00:01:10,874 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ca95584297d44cadac51e0a481551fc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715264867 2024-11-16T00:01:10,874 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbcab3aee0e743f0ad4399745bcc595a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715268555 2024-11-16T00:01:10,874 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a95d8cb79f24c5bbef803bc02894dbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731715265209 2024-11-16T00:01:10,874 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd6f956492b41c69a49328ffb49dbab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:10,875 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 09e5a38d3ece422ca5fd97b9b2923578, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731715266379 2024-11-16T00:01:10,875 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b1d736a97144699548dfe8385c5bcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715268555 2024-11-16T00:01:10,876 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5486427da0364db1ac1b92d7c4579ce6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:10,894 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:10,902 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#532 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:10,902 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/2806aef1af3e44578ad40f07e02f05c7 is 50, key is test_row_0/B:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,910 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116a333dc10582242ecb3427d12dbc81e1f_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:10,913 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116a333dc10582242ecb3427d12dbc81e1f_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:10,914 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a333dc10582242ecb3427d12dbc81e1f_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742452_1628 (size=12459) 2024-11-16T00:01:10,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742453_1629 (size=4469) 2024-11-16T00:01:10,935 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#531 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:10,936 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2546c1cbd7e84537b72f5a18f3574b14 is 175, key is test_row_0/A:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:10,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742454_1630 (size=31413) 2024-11-16T00:01:11,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T00:01:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:11,004 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T00:01:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116817c2fd694ca4c388b81121e369e3746_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715270385/Put/seqid=0 2024-11-16T00:01:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:11,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742455_1631 (size=12304) 2024-11-16T00:01:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715331051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715331051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715331051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715331056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715331059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715331170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715331170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715331170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715331170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715331170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,334 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/2806aef1af3e44578ad40f07e02f05c7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2806aef1af3e44578ad40f07e02f05c7 2024-11-16T00:01:11,344 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 2806aef1af3e44578ad40f07e02f05c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:11,344 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:11,344 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=10, startTime=1731715270868; duration=0sec 2024-11-16T00:01:11,344 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:11,344 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:11,344 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T00:01:11,346 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72409 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T00:01:11,346 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:11,346 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:11,346 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d57ad5141a714e70a96f78bb70986290, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=70.7 K 2024-11-16T00:01:11,346 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d57ad5141a714e70a96f78bb70986290, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731715264183 2024-11-16T00:01:11,347 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting a2ba0d3bdcf6447faeaf00588c608f41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731715264867 2024-11-16T00:01:11,347 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f89bab52157e4b4eb551a538d9eda226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731715265209 2024-11-16T00:01:11,347 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3c94b0ae2241beadbad6e3125ed819, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731715266379 2024-11-16T00:01:11,347 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3b7179072647e2932244f1c65e3996, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731715268555 2024-11-16T00:01:11,348 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b9e2f3627b45299ef0d3c9c18a39db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:11,353 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2546c1cbd7e84537b72f5a18f3574b14 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14 2024-11-16T00:01:11,357 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into 2546c1cbd7e84537b72f5a18f3574b14(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:11,357 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:11,357 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=10, startTime=1731715270867; duration=0sec 2024-11-16T00:01:11,357 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:11,357 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:11,359 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#534 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:11,360 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/07b0ef5c84f146c7a91f3b28b4f43060 is 50, key is test_row_0/C:col10/1731715269727/Put/seqid=0 2024-11-16T00:01:11,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742456_1632 (size=12459) 2024-11-16T00:01:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715331380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715331380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715331380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715331380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715331380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,386 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/07b0ef5c84f146c7a91f3b28b4f43060 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/07b0ef5c84f146c7a91f3b28b4f43060 2024-11-16T00:01:11,395 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into 07b0ef5c84f146c7a91f3b28b4f43060(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:11,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:11,395 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=10, startTime=1731715270868; duration=0sec 2024-11-16T00:01:11,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:11,395 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:11,435 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116817c2fd694ca4c388b81121e369e3746_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116817c2fd694ca4c388b81121e369e3746_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/5b084555c2d243f0a29fda9c8e5ff4d5, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/5b084555c2d243f0a29fda9c8e5ff4d5 is 175, key is test_row_0/A:col10/1731715270385/Put/seqid=0 2024-11-16T00:01:11,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742457_1633 (size=31105) 2024-11-16T00:01:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:11,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715331689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:11,854 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/5b084555c2d243f0a29fda9c8e5ff4d5 2024-11-16T00:01:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d656f8ddaa5f4587aec508eb8d985e69 is 50, key is test_row_0/B:col10/1731715270385/Put/seqid=0 2024-11-16T00:01:11,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742458_1634 (size=12151) 2024-11-16T00:01:12,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715332200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715332201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:12,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715332201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715332201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715332201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:12,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:12,270 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d656f8ddaa5f4587aec508eb8d985e69 2024-11-16T00:01:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/0c39d50f2c3840cb9e2f6d6605367a58 is 50, key is test_row_0/C:col10/1731715270385/Put/seqid=0 2024-11-16T00:01:12,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742459_1635 (size=12151) 2024-11-16T00:01:12,315 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/0c39d50f2c3840cb9e2f6d6605367a58 2024-11-16T00:01:12,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/5b084555c2d243f0a29fda9c8e5ff4d5 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5 2024-11-16T00:01:12,325 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5, entries=150, sequenceid=168, filesize=30.4 K 2024-11-16T00:01:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d656f8ddaa5f4587aec508eb8d985e69 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69 2024-11-16T00:01:12,330 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T00:01:12,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/0c39d50f2c3840cb9e2f6d6605367a58 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58 2024-11-16T00:01:12,336 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T00:01:12,338 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 895ec0f83f32ef97e348b295ffa56e5a in 1334ms, sequenceid=168, compaction requested=false 2024-11-16T00:01:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-16T00:01:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-16T00:01:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T00:01:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-16T00:01:12,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.8110 sec 2024-11-16T00:01:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T00:01:12,680 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:12,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-16T00:01:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T00:01:12,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:12,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:12,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T00:01:12,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-16T00:01:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:12,839 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T00:01:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116295fafbe373d4657a86ed663b3ab4af9_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715271049/Put/seqid=0 2024-11-16T00:01:12,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742460_1636 (size=12304) 2024-11-16T00:01:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T00:01:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:13,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:13,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715333226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715333228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715333234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715333234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715333236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:13,275 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116295fafbe373d4657a86ed663b3ab4af9_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116295fafbe373d4657a86ed663b3ab4af9_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/e309a885f4b040d99a5044d1f41f48f6, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:13,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/e309a885f4b040d99a5044d1f41f48f6 is 175, key is test_row_0/A:col10/1731715271049/Put/seqid=0 2024-11-16T00:01:13,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742461_1637 (size=31105) 2024-11-16T00:01:13,281 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/e309a885f4b040d99a5044d1f41f48f6 2024-11-16T00:01:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/740f29bcc9374c16bb8c6f8d465b05c0 is 50, key is test_row_0/B:col10/1731715271049/Put/seqid=0 2024-11-16T00:01:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742462_1638 (size=12151) 2024-11-16T00:01:13,299 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/740f29bcc9374c16bb8c6f8d465b05c0 2024-11-16T00:01:13,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3cee8bb6d81b4964a995946c32256867 is 50, key is test_row_0/C:col10/1731715271049/Put/seqid=0 2024-11-16T00:01:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T00:01:13,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742463_1639 (size=12151) 2024-11-16T00:01:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715333340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715333340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715333350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715333352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715333352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715333550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715333550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715333560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715333560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715333560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,716 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3cee8bb6d81b4964a995946c32256867 2024-11-16T00:01:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/e309a885f4b040d99a5044d1f41f48f6 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6 2024-11-16T00:01:13,739 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6, entries=150, sequenceid=191, filesize=30.4 K 2024-11-16T00:01:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/740f29bcc9374c16bb8c6f8d465b05c0 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0 2024-11-16T00:01:13,743 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-16T00:01:13,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3cee8bb6d81b4964a995946c32256867 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867 2024-11-16T00:01:13,747 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867, entries=150, sequenceid=191, filesize=11.9 K 2024-11-16T00:01:13,748 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 895ec0f83f32ef97e348b295ffa56e5a in 910ms, sequenceid=191, compaction requested=true 2024-11-16T00:01:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:13,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-16T00:01:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-16T00:01:13,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-16T00:01:13,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0650 sec 2024-11-16T00:01:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.0840 sec 2024-11-16T00:01:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T00:01:13,820 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:13,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-16T00:01:13,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:13,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:13,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:13,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:13,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:01:13,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160b7429c080984240b22bd4013eb85b4e_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:13,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742464_1640 (size=14794) 2024-11-16T00:01:13,891 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:13,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715333888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715333893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715333895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,902 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160b7429c080984240b22bd4013eb85b4e_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7429c080984240b22bd4013eb85b4e_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715333895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715333896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:13,903 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/63889eafc536418197f83a3eea5f8e8a, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:13,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/63889eafc536418197f83a3eea5f8e8a is 175, key is test_row_0/A:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:13,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742465_1641 (size=39749) 2024-11-16T00:01:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:13,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:13,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715334003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715334010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715334010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715334010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715334011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715334220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715334220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715334225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715334224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715334234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/63889eafc536418197f83a3eea5f8e8a 2024-11-16T00:01:14,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f37767fc59b140bf95b831d45fbacdea is 50, key is test_row_0/B:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742466_1642 (size=12151) 2024-11-16T00:01:14,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715334529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715334529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715334540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715334540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715334550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:14,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:14,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f37767fc59b140bf95b831d45fbacdea 2024-11-16T00:01:14,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:14,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/5f4f44820b974620bd5a025234b33a1e is 50, key is test_row_0/C:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:14,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742467_1643 (size=12151) 2024-11-16T00:01:14,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/5f4f44820b974620bd5a025234b33a1e 2024-11-16T00:01:14,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/63889eafc536418197f83a3eea5f8e8a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a 2024-11-16T00:01:14,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a, entries=200, sequenceid=208, filesize=38.8 K 2024-11-16T00:01:14,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f37767fc59b140bf95b831d45fbacdea as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea 2024-11-16T00:01:14,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T00:01:14,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/5f4f44820b974620bd5a025234b33a1e as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e 2024-11-16T00:01:14,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T00:01:14,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 895ec0f83f32ef97e348b295ffa56e5a in 954ms, sequenceid=208, compaction requested=true 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:14,817 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:01:14,817 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:01:14,819 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:01:14,819 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:14,820 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,820 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2806aef1af3e44578ad40f07e02f05c7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=47.8 K 2024-11-16T00:01:14,820 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:01:14,820 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:14,820 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,820 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=130.2 K 2024-11-16T00:01:14,820 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,820 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a] 2024-11-16T00:01:14,820 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2806aef1af3e44578ad40f07e02f05c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:14,820 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2546c1cbd7e84537b72f5a18f3574b14, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:14,821 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting d656f8ddaa5f4587aec508eb8d985e69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731715270382 2024-11-16T00:01:14,821 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b084555c2d243f0a29fda9c8e5ff4d5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731715270382 2024-11-16T00:01:14,821 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting e309a885f4b040d99a5044d1f41f48f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731715271049 2024-11-16T00:01:14,821 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 740f29bcc9374c16bb8c6f8d465b05c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731715271049 2024-11-16T00:01:14,823 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63889eafc536418197f83a3eea5f8e8a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:14,824 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f37767fc59b140bf95b831d45fbacdea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:14,834 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:14,856 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#544 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:14,856 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/5cbacd7ec6954eefa039734113c5ef04 is 50, key is test_row_0/B:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:14,858 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ac13afbc94fd4c2abece8e2b63f961f8_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:14,860 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ac13afbc94fd4c2abece8e2b63f961f8_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:14,861 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ac13afbc94fd4c2abece8e2b63f961f8_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:14,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742468_1644 (size=12595) 2024-11-16T00:01:14,879 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/5cbacd7ec6954eefa039734113c5ef04 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5cbacd7ec6954eefa039734113c5ef04 2024-11-16T00:01:14,890 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 5cbacd7ec6954eefa039734113c5ef04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:14,890 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:14,890 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=12, startTime=1731715274817; duration=0sec 2024-11-16T00:01:14,890 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:14,890 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:14,890 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T00:01:14,892 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T00:01:14,892 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:14,892 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,892 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/07b0ef5c84f146c7a91f3b28b4f43060, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=47.8 K 2024-11-16T00:01:14,892 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b0ef5c84f146c7a91f3b28b4f43060, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731715269722 2024-11-16T00:01:14,893 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c39d50f2c3840cb9e2f6d6605367a58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731715270382 2024-11-16T00:01:14,893 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cee8bb6d81b4964a995946c32256867, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731715271049 2024-11-16T00:01:14,893 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4f44820b974620bd5a025234b33a1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:14,901 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:14,902 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/57ce14a46bad4afaa3fb8911d68375a2 is 50, key is test_row_0/C:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:14,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742469_1645 (size=4469) 2024-11-16T00:01:14,922 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#543 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:14,922 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0b8dfaa75e546809f28bc5fc32cc275 is 175, key is test_row_0/A:col10/1731715273224/Put/seqid=0 2024-11-16T00:01:14,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-16T00:01:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:14,926 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:01:14,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742470_1646 (size=12595) 2024-11-16T00:01:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116679ec266fdf14f1aa23634158a56502a_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715273895/Put/seqid=0 2024-11-16T00:01:14,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742471_1647 (size=31549) 2024-11-16T00:01:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:14,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742472_1648 (size=12304) 2024-11-16T00:01:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:14,991 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116679ec266fdf14f1aa23634158a56502a_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116679ec266fdf14f1aa23634158a56502a_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:14,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/942b897d2928474ab9e27d799bb12e8f, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/942b897d2928474ab9e27d799bb12e8f is 175, key is test_row_0/A:col10/1731715273895/Put/seqid=0 2024-11-16T00:01:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742473_1649 (size=31105) 2024-11-16T00:01:15,007 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/942b897d2928474ab9e27d799bb12e8f 2024-11-16T00:01:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f8a5fe77caf54474bf99c42842effcb4 is 50, key is test_row_0/B:col10/1731715273895/Put/seqid=0 2024-11-16T00:01:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742474_1650 (size=12151) 2024-11-16T00:01:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:15,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715335059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715335060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715335063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715335064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715335066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715335169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715335180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715335180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715335180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,361 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/b0b8dfaa75e546809f28bc5fc32cc275 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275 2024-11-16T00:01:15,366 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into b0b8dfaa75e546809f28bc5fc32cc275(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:15,366 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:15,366 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=12, startTime=1731715274817; duration=0sec 2024-11-16T00:01:15,366 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:15,366 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:15,369 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/57ce14a46bad4afaa3fb8911d68375a2 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/57ce14a46bad4afaa3fb8911d68375a2 2024-11-16T00:01:15,373 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into 57ce14a46bad4afaa3fb8911d68375a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:15,373 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:15,373 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=12, startTime=1731715274817; duration=0sec 2024-11-16T00:01:15,373 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:15,373 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:15,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715335380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715335390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715335390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715335393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,436 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f8a5fe77caf54474bf99c42842effcb4 2024-11-16T00:01:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f6ef8e85918c48348f15e79ce00ea8e7 is 50, key is test_row_0/C:col10/1731715273895/Put/seqid=0 2024-11-16T00:01:15,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742475_1651 (size=12151) 2024-11-16T00:01:15,451 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f6ef8e85918c48348f15e79ce00ea8e7 2024-11-16T00:01:15,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/942b897d2928474ab9e27d799bb12e8f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f 2024-11-16T00:01:15,461 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f, entries=150, sequenceid=228, filesize=30.4 K 2024-11-16T00:01:15,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/f8a5fe77caf54474bf99c42842effcb4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4 2024-11-16T00:01:15,468 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4, entries=150, sequenceid=228, filesize=11.9 K 2024-11-16T00:01:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/f6ef8e85918c48348f15e79ce00ea8e7 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7 2024-11-16T00:01:15,474 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7, entries=150, sequenceid=228, filesize=11.9 K 2024-11-16T00:01:15,475 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 895ec0f83f32ef97e348b295ffa56e5a in 549ms, sequenceid=228, compaction requested=false 2024-11-16T00:01:15,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:15,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:15,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-16T00:01:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-16T00:01:15,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-16T00:01:15,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6540 sec 2024-11-16T00:01:15,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.6610 sec 2024-11-16T00:01:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:15,697 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:15,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:01:15,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cc36c98597de4f97a188f9663986eed5_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742476_1652 (size=12304) 2024-11-16T00:01:15,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715335730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715335734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715335734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715335738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715335850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715335851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715335851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715335851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T00:01:15,990 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:15,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-16T00:01:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:15,996 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:15,997 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:15,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:16,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715336060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715336061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715336061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715336062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715336070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,072 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 895ec0f83f32ef97e348b295ffa56e5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T00:01:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:16,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:16,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cc36c98597de4f97a188f9663986eed5_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cc36c98597de4f97a188f9663986eed5_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:16,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/24110c685a2e4607a43e027756c6876d, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/24110c685a2e4607a43e027756c6876d is 175, key is test_row_0/A:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:16,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742477_1653 (size=31105) 2024-11-16T00:01:16,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-16T00:01:16,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:16,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-16T00:01:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715336370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715336372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715336380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715336380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-16T00:01:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/24110c685a2e4607a43e027756c6876d 2024-11-16T00:01:16,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6383cc5deb1442048e4126c4b84df8c1 is 50, key is test_row_0/B:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:16,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742478_1654 (size=12151) 2024-11-16T00:01:16,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6383cc5deb1442048e4126c4b84df8c1 2024-11-16T00:01:16,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/c469351a77284f0f9623a371853e8083 is 50, key is test_row_0/C:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:16,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742479_1655 (size=12151) 2024-11-16T00:01:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:16,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/c469351a77284f0f9623a371853e8083 2024-11-16T00:01:16,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/24110c685a2e4607a43e027756c6876d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d 2024-11-16T00:01:16,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d, entries=150, sequenceid=248, filesize=30.4 K 2024-11-16T00:01:16,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6383cc5deb1442048e4126c4b84df8c1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1 2024-11-16T00:01:16,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-16T00:01:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:16,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T00:01:16,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/c469351a77284f0f9623a371853e8083 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083 2024-11-16T00:01:16,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083, entries=150, sequenceid=248, filesize=11.9 K 2024-11-16T00:01:16,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 895ec0f83f32ef97e348b295ffa56e5a in 956ms, sequenceid=248, compaction requested=true 2024-11-16T00:01:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:16,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T00:01:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T00:01:16,655 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:16,656 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:16,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:16,657 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,657 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=91.6 K 2024-11-16T00:01:16,657 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d] 2024-11-16T00:01:16,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b8dfaa75e546809f28bc5fc32cc275, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:16,657 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:16,657 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 942b897d2928474ab9e27d799bb12e8f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731715273886 2024-11-16T00:01:16,659 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24110c685a2e4607a43e027756c6876d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:16,661 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:16,661 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:16,661 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,661 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/57ce14a46bad4afaa3fb8911d68375a2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.0 K 2024-11-16T00:01:16,662 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ce14a46bad4afaa3fb8911d68375a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:16,662 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ef8e85918c48348f15e79ce00ea8e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731715273886 2024-11-16T00:01:16,662 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting c469351a77284f0f9623a371853e8083, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:16,671 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,672 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:16,673 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/10e327444aad4afea9d2965dd187ef8c is 50, key is test_row_0/C:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:16,674 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165ca58e88b6d6465bbe71a376b95af955_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165ca58e88b6d6465bbe71a376b95af955_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,676 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165ca58e88b6d6465bbe71a376b95af955_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742480_1656 (size=12697) 2024-11-16T00:01:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742481_1657 (size=4469) 2024-11-16T00:01:16,711 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#553 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:16,712 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf is 175, key is test_row_0/A:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:16,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742482_1658 (size=31651) 2024-11-16T00:01:16,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-16T00:01:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T00:01:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165497ad65712846ca946f752664d945e1_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715275733/Put/seqid=0 2024-11-16T00:01:16,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742483_1659 (size=12454) 2024-11-16T00:01:16,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:16,838 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165497ad65712846ca946f752664d945e1_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165497ad65712846ca946f752664d945e1_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:16,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d533fe3b834640daac955d8765e88119, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:16,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d533fe3b834640daac955d8765e88119 is 175, key is test_row_0/A:col10/1731715275733/Put/seqid=0 2024-11-16T00:01:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:16,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:16,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742484_1660 (size=31255) 2024-11-16T00:01:16,890 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d533fe3b834640daac955d8765e88119 2024-11-16T00:01:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/519137d4203848ec802dd5bc79ce0381 is 50, key is test_row_0/B:col10/1731715275733/Put/seqid=0 2024-11-16T00:01:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715336920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715336922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715336926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715336930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742485_1661 (size=12301) 2024-11-16T00:01:16,957 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/519137d4203848ec802dd5bc79ce0381 2024-11-16T00:01:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/9f813f0272564eea84f9e5feac443c26 is 50, key is test_row_0/C:col10/1731715275733/Put/seqid=0 2024-11-16T00:01:16,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742486_1662 (size=12301) 2024-11-16T00:01:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715337040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715337040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715337040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715337041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:17,157 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/10e327444aad4afea9d2965dd187ef8c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/10e327444aad4afea9d2965dd187ef8c 2024-11-16T00:01:17,158 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf 2024-11-16T00:01:17,176 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into fd12fd1dbb9f4f1a8e6a086f52dc00cf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:17,176 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:17,176 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=13, startTime=1731715276653; duration=0sec 2024-11-16T00:01:17,176 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:17,176 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:17,176 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:17,177 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:17,177 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:17,177 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:17,177 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5cbacd7ec6954eefa039734113c5ef04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.0 K 2024-11-16T00:01:17,178 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbacd7ec6954eefa039734113c5ef04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731715273224 2024-11-16T00:01:17,179 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a5fe77caf54474bf99c42842effcb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731715273886 2024-11-16T00:01:17,179 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6383cc5deb1442048e4126c4b84df8c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:17,192 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into 10e327444aad4afea9d2965dd187ef8c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:17,192 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:17,192 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=13, startTime=1731715276654; duration=0sec 2024-11-16T00:01:17,192 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:17,192 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:17,196 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:17,196 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/048eb6a515d14e6897b711fa158e3bf1 is 50, key is test_row_0/B:col10/1731715275061/Put/seqid=0 2024-11-16T00:01:17,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742487_1663 (size=12697) 2024-11-16T00:01:17,208 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/048eb6a515d14e6897b711fa158e3bf1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/048eb6a515d14e6897b711fa158e3bf1 2024-11-16T00:01:17,215 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 048eb6a515d14e6897b711fa158e3bf1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:17,215 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:17,215 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=13, startTime=1731715276654; duration=0sec 2024-11-16T00:01:17,215 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:17,215 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:17,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715337250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715337250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715337250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715337260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,381 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/9f813f0272564eea84f9e5feac443c26 2024-11-16T00:01:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d533fe3b834640daac955d8765e88119 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119 2024-11-16T00:01:17,396 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119, entries=150, sequenceid=266, filesize=30.5 K 2024-11-16T00:01:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/519137d4203848ec802dd5bc79ce0381 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381 2024-11-16T00:01:17,402 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T00:01:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/9f813f0272564eea84f9e5feac443c26 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26 2024-11-16T00:01:17,405 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T00:01:17,406 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 895ec0f83f32ef97e348b295ffa56e5a in 612ms, sequenceid=266, compaction requested=false 2024-11-16T00:01:17,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:17,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:17,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-16T00:01:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-16T00:01:17,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-16T00:01:17,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-16T00:01:17,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.4200 sec 2024-11-16T00:01:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:17,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T00:01:17,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fad08052fedb474080d64d0ead760445_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:17,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742488_1664 (size=14994) 2024-11-16T00:01:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715337583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:17,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fad08052fedb474080d64d0ead760445_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fad08052fedb474080d64d0ead760445_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715337585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715337586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715337587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2e1b2225d5134e01972d786c97abc85c, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:17,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2e1b2225d5134e01972d786c97abc85c is 175, key is test_row_0/A:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:17,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742489_1665 (size=39949) 2024-11-16T00:01:17,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2e1b2225d5134e01972d786c97abc85c 2024-11-16T00:01:17,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ad5de1fea1194050971c7e82a25f274f is 50, key is test_row_0/B:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:17,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742490_1666 (size=12301) 2024-11-16T00:01:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715337689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715337700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715337700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715337715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715337900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715337910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715337911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:17,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715337929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ad5de1fea1194050971c7e82a25f274f 2024-11-16T00:01:18,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/993690333e7244a8b8d88751800ce43b is 50, key is test_row_0/C:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:18,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742491_1667 (size=12301) 2024-11-16T00:01:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715338089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,093 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 895ec0f83f32ef97e348b295ffa56e5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T00:01:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T00:01:18,140 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:18,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-16T00:01:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-16T00:01:18,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:18,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:18,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715338209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715338220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715338220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715338239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-16T00:01:18,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-16T00:01:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-16T00:01:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-16T00:01:18,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/993690333e7244a8b8d88751800ce43b 2024-11-16T00:01:18,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/2e1b2225d5134e01972d786c97abc85c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c 2024-11-16T00:01:18,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c, entries=200, sequenceid=288, filesize=39.0 K 2024-11-16T00:01:18,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/ad5de1fea1194050971c7e82a25f274f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f 2024-11-16T00:01:18,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T00:01:18,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/993690333e7244a8b8d88751800ce43b as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b 2024-11-16T00:01:18,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T00:01:18,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 895ec0f83f32ef97e348b295ffa56e5a in 933ms, sequenceid=288, compaction requested=true 2024-11-16T00:01:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:18,498 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:18,498 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:18,499 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:18,499 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:18,499 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,499 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=100.4 K 2024-11-16T00:01:18,499 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c] 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:18,500 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd12fd1dbb9f4f1a8e6a086f52dc00cf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:18,500 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/048eb6a515d14e6897b711fa158e3bf1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.4 K 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting d533fe3b834640daac955d8765e88119, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731715275722 2024-11-16T00:01:18,500 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 048eb6a515d14e6897b711fa158e3bf1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:18,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 519137d4203848ec802dd5bc79ce0381, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731715275722 2024-11-16T00:01:18,501 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1b2225d5134e01972d786c97abc85c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276909 2024-11-16T00:01:18,501 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting ad5de1fea1194050971c7e82a25f274f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276919 2024-11-16T00:01:18,510 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:18,511 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:18,512 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/9b82fb37cdda4719b010d4452b7bef75 is 50, key is test_row_0/B:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:18,514 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164251faaf81c74f659ba004a6d9b276cd_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:18,516 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164251faaf81c74f659ba004a6d9b276cd_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:18,516 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164251faaf81c74f659ba004a6d9b276cd_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:18,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742492_1668 (size=12949) 2024-11-16T00:01:18,532 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/9b82fb37cdda4719b010d4452b7bef75 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9b82fb37cdda4719b010d4452b7bef75 2024-11-16T00:01:18,536 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 9b82fb37cdda4719b010d4452b7bef75(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:18,536 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:18,536 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=13, startTime=1731715278498; duration=0sec 2024-11-16T00:01:18,536 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:18,536 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:18,537 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:18,538 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:18,538 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:18,538 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,538 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/10e327444aad4afea9d2965dd187ef8c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.4 K 2024-11-16T00:01:18,538 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e327444aad4afea9d2965dd187ef8c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731715275061 2024-11-16T00:01:18,539 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f813f0272564eea84f9e5feac443c26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731715275722 2024-11-16T00:01:18,539 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 993690333e7244a8b8d88751800ce43b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276919 2024-11-16T00:01:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742493_1669 (size=4469) 2024-11-16T00:01:18,545 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#561 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:18,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/477716f102004bf380a271f42bc91f86 is 175, key is test_row_0/A:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:18,549 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#563 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:18,550 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/18c8a37fa77b4dd78945152cccf7e278 is 50, key is test_row_0/C:col10/1731715277563/Put/seqid=0 2024-11-16T00:01:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742494_1670 (size=31903) 2024-11-16T00:01:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742495_1671 (size=12949) 2024-11-16T00:01:18,589 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/477716f102004bf380a271f42bc91f86 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86 2024-11-16T00:01:18,593 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/18c8a37fa77b4dd78945152cccf7e278 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/18c8a37fa77b4dd78945152cccf7e278 2024-11-16T00:01:18,594 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into 477716f102004bf380a271f42bc91f86(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:18,594 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:18,595 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=13, startTime=1731715278497; duration=0sec 2024-11-16T00:01:18,595 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:18,595 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:18,599 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into 18c8a37fa77b4dd78945152cccf7e278(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:18,599 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:18,599 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=13, startTime=1731715278498; duration=0sec 2024-11-16T00:01:18,599 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:18,599 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:18,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-16T00:01:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,602 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T00:01:18,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686d55948501540f6a67f013632450197_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715277579/Put/seqid=0 2024-11-16T00:01:18,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742496_1672 (size=12454) 2024-11-16T00:01:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,617 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686d55948501540f6a67f013632450197_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686d55948501540f6a67f013632450197_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/c7e4a3e098f046cf917796ca6d0e75be, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/c7e4a3e098f046cf917796ca6d0e75be is 175, key is test_row_0/A:col10/1731715277579/Put/seqid=0 2024-11-16T00:01:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742497_1673 (size=31255) 2024-11-16T00:01:18,623 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/c7e4a3e098f046cf917796ca6d0e75be 2024-11-16T00:01:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/e1a30d6de1f94d3799e78e5dadd18943 is 50, key is test_row_0/B:col10/1731715277579/Put/seqid=0 2024-11-16T00:01:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742498_1674 (size=12301) 2024-11-16T00:01:18,634 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/e1a30d6de1f94d3799e78e5dadd18943 2024-11-16T00:01:18,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/7483107449a046febf5ea9f61692e8ad is 50, key is test_row_0/C:col10/1731715277579/Put/seqid=0 2024-11-16T00:01:18,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742499_1675 (size=12301) 2024-11-16T00:01:18,645 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/7483107449a046febf5ea9f61692e8ad 2024-11-16T00:01:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/c7e4a3e098f046cf917796ca6d0e75be as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be 2024-11-16T00:01:18,659 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be, entries=150, sequenceid=308, filesize=30.5 K 2024-11-16T00:01:18,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/e1a30d6de1f94d3799e78e5dadd18943 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943 2024-11-16T00:01:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,662 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T00:01:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/7483107449a046febf5ea9f61692e8ad as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad 2024-11-16T00:01:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T00:01:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,667 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 895ec0f83f32ef97e348b295ffa56e5a in 65ms, sequenceid=308, compaction requested=false 2024-11-16T00:01:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-16T00:01:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-16T00:01:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 526 msec 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 530 msec 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-16T00:01:18,770 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,770 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:18,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T00:01:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-16T00:01:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,772 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,773 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:18,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f88eff79bf6441ec9e1edd32a32531bc_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742501_1677 (size=27698) 2024-11-16T00:01:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715338805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715338808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715338809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715338810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715338920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715338920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715338920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715338920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:18,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:18,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715339130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715339130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715339130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715339131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,194 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:19,197 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f88eff79bf6441ec9e1edd32a32531bc_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f88eff79bf6441ec9e1edd32a32531bc_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:19,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/dba300b487b0475497d1c4d0c9cc1877, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:19,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/dba300b487b0475497d1c4d0c9cc1877 is 175, key is test_row_0/A:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:19,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742500_1676 (size=83485) 2024-11-16T00:01:19,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715339440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715339440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715339440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715339450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/dba300b487b0475497d1c4d0c9cc1877 2024-11-16T00:01:19,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b7220d2575341ada6a0b56459393edc is 50, key is test_row_0/B:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:19,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742502_1678 (size=12301) 2024-11-16T00:01:19,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:19,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:19,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715339950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715339950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715339960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:19,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715339960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:20,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b7220d2575341ada6a0b56459393edc 2024-11-16T00:01:20,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/2462585e471b4f0c8b9c7911a5430e04 is 50, key is test_row_0/C:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:20,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742503_1679 (size=12301) 2024-11-16T00:01:20,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:20,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/2462585e471b4f0c8b9c7911a5430e04 2024-11-16T00:01:20,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/dba300b487b0475497d1c4d0c9cc1877 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877 2024-11-16T00:01:20,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877, entries=450, sequenceid=320, filesize=81.5 K 2024-11-16T00:01:20,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/4b7220d2575341ada6a0b56459393edc as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc 2024-11-16T00:01:20,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T00:01:20,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/2462585e471b4f0c8b9c7911a5430e04 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04 2024-11-16T00:01:20,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T00:01:20,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 895ec0f83f32ef97e348b295ffa56e5a in 1348ms, sequenceid=320, compaction requested=true 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:20,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T00:01:20,120 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:20,120 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:20,122 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:20,122 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:20,122 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 146643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:20,122 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,122 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:20,123 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/18c8a37fa77b4dd78945152cccf7e278, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.7 K 2024-11-16T00:01:20,123 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,123 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=143.2 K 2024-11-16T00:01:20,123 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,123 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877] 2024-11-16T00:01:20,123 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c8a37fa77b4dd78945152cccf7e278, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276919 2024-11-16T00:01:20,123 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477716f102004bf380a271f42bc91f86, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276919 2024-11-16T00:01:20,124 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 7483107449a046febf5ea9f61692e8ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731715277579 2024-11-16T00:01:20,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e4a3e098f046cf917796ca6d0e75be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731715277579 2024-11-16T00:01:20,124 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2462585e471b4f0c8b9c7911a5430e04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:20,124 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba300b487b0475497d1c4d0c9cc1877, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:20,130 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:20,131 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#570 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:20,131 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/fcebfe0a588a4758be9a21c1d698076d is 50, key is test_row_0/C:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:20,133 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111659ee9e26d963461ba6804273cbc08535_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:20,135 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111659ee9e26d963461ba6804273cbc08535_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:20,136 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111659ee9e26d963461ba6804273cbc08535_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742504_1680 (size=13051) 2024-11-16T00:01:20,157 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/fcebfe0a588a4758be9a21c1d698076d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/fcebfe0a588a4758be9a21c1d698076d 2024-11-16T00:01:20,171 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into fcebfe0a588a4758be9a21c1d698076d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:20,171 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:20,171 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=13, startTime=1731715280119; duration=0sec 2024-11-16T00:01:20,171 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:20,171 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:20,171 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742505_1681 (size=4469) 2024-11-16T00:01:20,177 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:20,177 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:20,177 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,178 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9b82fb37cdda4719b010d4452b7bef75, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.7 K 2024-11-16T00:01:20,178 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#571 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:20,179 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/f5433fc7ef1f4df3bb4312465ae6102d is 175, key is test_row_0/A:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:20,179 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b82fb37cdda4719b010d4452b7bef75, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731715276919 2024-11-16T00:01:20,180 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a30d6de1f94d3799e78e5dadd18943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731715277579 2024-11-16T00:01:20,180 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7220d2575341ada6a0b56459393edc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:20,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-16T00:01:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T00:01:20,221 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#572 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:20,222 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/46725def8844401e8b5648243885f7f9 is 50, key is test_row_0/B:col10/1731715278770/Put/seqid=0 2024-11-16T00:01:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742506_1682 (size=32005) 2024-11-16T00:01:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742507_1683 (size=13051) 2024-11-16T00:01:20,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665b135da3c104982bedbe695217909f5_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715278797/Put/seqid=0 2024-11-16T00:01:20,301 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/46725def8844401e8b5648243885f7f9 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/46725def8844401e8b5648243885f7f9 2024-11-16T00:01:20,327 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 46725def8844401e8b5648243885f7f9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:20,327 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:20,327 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=13, startTime=1731715280119; duration=0sec 2024-11-16T00:01:20,327 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:20,327 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742508_1684 (size=12454) 2024-11-16T00:01:20,645 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/f5433fc7ef1f4df3bb4312465ae6102d as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d 2024-11-16T00:01:20,689 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into f5433fc7ef1f4df3bb4312465ae6102d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:20,689 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:20,689 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=13, startTime=1731715280119; duration=0sec 2024-11-16T00:01:20,690 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:20,690 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:20,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665b135da3c104982bedbe695217909f5_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665b135da3c104982bedbe695217909f5_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/0d57a21a9196476088acd3abf0f1b40f, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/0d57a21a9196476088acd3abf0f1b40f is 175, key is test_row_0/A:col10/1731715278797/Put/seqid=0 2024-11-16T00:01:20,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742509_1685 (size=31255) 2024-11-16T00:01:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715340985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:20,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715340987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715340987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:20,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715340989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:20,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715341100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715341100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715341100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715341101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,187 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/0d57a21a9196476088acd3abf0f1b40f 2024-11-16T00:01:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/15be7b27d4e04d51a4b5012602560f0c is 50, key is test_row_0/B:col10/1731715278797/Put/seqid=0 2024-11-16T00:01:21,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742510_1686 (size=12301) 2024-11-16T00:01:21,225 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/15be7b27d4e04d51a4b5012602560f0c 2024-11-16T00:01:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1111b1dcc4194f6fa5d9947374657496 is 50, key is test_row_0/C:col10/1731715278797/Put/seqid=0 2024-11-16T00:01:21,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742511_1687 (size=12301) 2024-11-16T00:01:21,298 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1111b1dcc4194f6fa5d9947374657496 2024-11-16T00:01:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/0d57a21a9196476088acd3abf0f1b40f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f 2024-11-16T00:01:21,306 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f, entries=150, sequenceid=345, filesize=30.5 K 2024-11-16T00:01:21,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/15be7b27d4e04d51a4b5012602560f0c as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c 2024-11-16T00:01:21,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715341310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,314 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T00:01:21,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715341310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715341311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715341313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1111b1dcc4194f6fa5d9947374657496 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,322 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T00:01:21,323 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 895ec0f83f32ef97e348b295ffa56e5a in 1118ms, sequenceid=345, compaction requested=false 2024-11-16T00:01:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-16T00:01:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-16T00:01:21,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-16T00:01:21,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5510 sec 2024-11-16T00:01:21,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 2.5590 sec 2024-11-16T00:01:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:21,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:21,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T00:01:21,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162779f69228dc4d1cb9eeb686af91b317_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:21,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742512_1688 (size=12454) 2024-11-16T00:01:21,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715341693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715341696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715341698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715341700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715341810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715341810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715341810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715341810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715342020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715342020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715342020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715342020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:22,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162779f69228dc4d1cb9eeb686af91b317_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162779f69228dc4d1cb9eeb686af91b317_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:22,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/74e2abcede57486299bfc2bb99a427f8, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/74e2abcede57486299bfc2bb99a427f8 is 175, key is test_row_0/A:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742513_1689 (size=31255) 2024-11-16T00:01:22,080 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/74e2abcede57486299bfc2bb99a427f8 2024-11-16T00:01:22,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/2a03b94ad0474a0ca050d5b480e51aee is 50, key is test_row_0/B:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742514_1690 (size=12301) 2024-11-16T00:01:22,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/2a03b94ad0474a0ca050d5b480e51aee 2024-11-16T00:01:22,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3e2061943af3411287960a61dba90fca is 50, key is test_row_0/C:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:22,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742515_1691 (size=12301) 2024-11-16T00:01:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34274 deadline: 1731715342130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,135 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0801813e30ae,43013,1731715110098 for 'test_row_0' in 895ec0f83f32ef97e348b295ffa56e5a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T00:01:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715342329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715342330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715342330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715342330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3e2061943af3411287960a61dba90fca 2024-11-16T00:01:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/74e2abcede57486299bfc2bb99a427f8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8 2024-11-16T00:01:22,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8, entries=150, sequenceid=364, filesize=30.5 K 2024-11-16T00:01:22,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/2a03b94ad0474a0ca050d5b480e51aee as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee 2024-11-16T00:01:22,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee, entries=150, sequenceid=364, filesize=12.0 K 2024-11-16T00:01:22,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/3e2061943af3411287960a61dba90fca as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca 2024-11-16T00:01:22,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca, entries=150, sequenceid=364, filesize=12.0 K 2024-11-16T00:01:22,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 895ec0f83f32ef97e348b295ffa56e5a in 915ms, sequenceid=364, compaction requested=true 2024-11-16T00:01:22,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:22,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T00:01:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:22,540 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:22,540 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T00:01:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 895ec0f83f32ef97e348b295ffa56e5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T00:01:22,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:22,545 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:22,545 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/A is initiating minor compaction (all files) 2024-11-16T00:01:22,545 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/A in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:22,546 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=92.3 K 2024-11-16T00:01:22,546 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:22,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. files: [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8] 2024-11-16T00:01:22,546 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:22,546 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/B is initiating minor compaction (all files) 2024-11-16T00:01:22,546 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/B in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:22,546 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/46725def8844401e8b5648243885f7f9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.8 K 2024-11-16T00:01:22,546 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5433fc7ef1f4df3bb4312465ae6102d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:22,547 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 46725def8844401e8b5648243885f7f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:22,547 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d57a21a9196476088acd3abf0f1b40f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731715278797 2024-11-16T00:01:22,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 15be7b27d4e04d51a4b5012602560f0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731715278797 2024-11-16T00:01:22,548 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e2abcede57486299bfc2bb99a427f8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731715280978 2024-11-16T00:01:22,548 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a03b94ad0474a0ca050d5b480e51aee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731715280978 2024-11-16T00:01:22,572 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#B#compaction#579 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:22,572 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,573 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/9fe924d5a4204b058f54cff9f5d6b4ce is 50, key is test_row_0/B:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:22,579 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411160505783921ac46809cd792d26e496fb1_895ec0f83f32ef97e348b295ffa56e5a store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,581 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411160505783921ac46809cd792d26e496fb1_895ec0f83f32ef97e348b295ffa56e5a, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,581 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160505783921ac46809cd792d26e496fb1_895ec0f83f32ef97e348b295ffa56e5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742516_1692 (size=13153) 2024-11-16T00:01:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742517_1693 (size=4469) 2024-11-16T00:01:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T00:01:22,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T00:01:22,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715342893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715342894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f46a1759f39240fbb4cc1f81b2f1f259_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715282844/Put/seqid=0 2024-11-16T00:01:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715342900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715342905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:22,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742518_1694 (size=12454) 2024-11-16T00:01:22,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:22,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f46a1759f39240fbb4cc1f81b2f1f259_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f46a1759f39240fbb4cc1f81b2f1f259_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:22,939 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d0bcdbc5c2b84d44824aae43b725cd13, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:22,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d0bcdbc5c2b84d44824aae43b725cd13 is 175, key is test_row_0/A:col10/1731715282844/Put/seqid=0 2024-11-16T00:01:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-16T00:01:22,949 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:22,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-16T00:01:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-11-16T00:01:22,968 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T00:01:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:22,977 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T00:01:22,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T00:01:22,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742519_1695 (size=31255) 2024-11-16T00:01:23,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715343014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715343011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715343022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715343022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,042 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#A#compaction#580 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:23,042 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7885e4f9d41d4b20b6c782df54dff123 is 175, key is test_row_0/A:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:23,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742520_1696 (size=32107) 2024-11-16T00:01:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:23,097 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/9fe924d5a4204b058f54cff9f5d6b4ce as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9fe924d5a4204b058f54cff9f5d6b4ce 2024-11-16T00:01:23,112 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/B of 895ec0f83f32ef97e348b295ffa56e5a into 9fe924d5a4204b058f54cff9f5d6b4ce(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:23,112 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:23,112 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/B, priority=13, startTime=1731715282540; duration=0sec 2024-11-16T00:01:23,112 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T00:01:23,112 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:B 2024-11-16T00:01:23,112 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T00:01:23,113 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T00:01:23,113 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1541): 895ec0f83f32ef97e348b295ffa56e5a/C is initiating minor compaction (all files) 2024-11-16T00:01:23,113 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 895ec0f83f32ef97e348b295ffa56e5a/C in TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,114 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/fcebfe0a588a4758be9a21c1d698076d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca] into tmpdir=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp, totalSize=36.8 K 2024-11-16T00:01:23,117 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting fcebfe0a588a4758be9a21c1d698076d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731715278728 2024-11-16T00:01:23,121 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 1111b1dcc4194f6fa5d9947374657496, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731715278797 2024-11-16T00:01:23,125 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2061943af3411287960a61dba90fca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731715280978 2024-11-16T00:01:23,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,157 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 895ec0f83f32ef97e348b295ffa56e5a#C#compaction#582 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T00:01:23,161 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/2ad4d418361b4e298ccceb0ffe23d360 is 50, key is test_row_0/C:col10/1731715281624/Put/seqid=0 2024-11-16T00:01:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742521_1697 (size=13153) 2024-11-16T00:01:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,237 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/2ad4d418361b4e298ccceb0ffe23d360 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2ad4d418361b4e298ccceb0ffe23d360 2024-11-16T00:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715343234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715343234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715343240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715343240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,258 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/C of 895ec0f83f32ef97e348b295ffa56e5a into 2ad4d418361b4e298ccceb0ffe23d360(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:23,258 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:23,258 INFO [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/C, priority=13, startTime=1731715282540; duration=0sec 2024-11-16T00:01:23,258 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:23,258 DEBUG [RS:0;0801813e30ae:43013-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:C 2024-11-16T00:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:23,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d0bcdbc5c2b84d44824aae43b725cd13 2024-11-16T00:01:23,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6480e05412d44edaa4af3b9dd55ad0bb is 50, key is test_row_0/B:col10/1731715282844/Put/seqid=0 2024-11-16T00:01:23,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742522_1698 (size=12301) 2024-11-16T00:01:23,522 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/7885e4f9d41d4b20b6c782df54dff123 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7885e4f9d41d4b20b6c782df54dff123 2024-11-16T00:01:23,533 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 895ec0f83f32ef97e348b295ffa56e5a/A of 895ec0f83f32ef97e348b295ffa56e5a into 7885e4f9d41d4b20b6c782df54dff123(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T00:01:23,533 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:23,533 INFO [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., storeName=895ec0f83f32ef97e348b295ffa56e5a/A, priority=13, startTime=1731715282539; duration=0sec 2024-11-16T00:01:23,533 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T00:01:23,533 DEBUG [RS:0;0801813e30ae:43013-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 895ec0f83f32ef97e348b295ffa56e5a:A 2024-11-16T00:01:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715343550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715343550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715343553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715343553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:23,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,813 INFO [Thread-2622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2622. 2024-11-16T00:01:23,813 DEBUG [Thread-2622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:23,813 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,813 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:23,815 INFO [Thread-2630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2630. 2024-11-16T00:01:23,815 DEBUG [Thread-2630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:23,815 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,815 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:23,816 INFO [Thread-2628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2628. 2024-11-16T00:01:23,816 DEBUG [Thread-2628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:23,816 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,817 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:23,817 INFO [Thread-2624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2624. 2024-11-16T00:01:23,817 DEBUG [Thread-2624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:23,817 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,817 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:23,818 INFO [Thread-2626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2626. 2024-11-16T00:01:23,818 DEBUG [Thread-2626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:23,818 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,818 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:23,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:23,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6480e05412d44edaa4af3b9dd55ad0bb 2024-11-16T00:01:23,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/445a6d40ac31495f823fe4ce56ea1771 is 50, key is test_row_0/C:col10/1731715282844/Put/seqid=0 2024-11-16T00:01:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742523_1699 (size=12301) 2024-11-16T00:01:23,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:23,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34260 deadline: 1731715344070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34290 deadline: 1731715344071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34226 deadline: 1731715344071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T00:01:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43013 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34242 deadline: 1731715344071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 2024-11-16T00:01:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 , the old value is region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T00:01:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., hostname=0801813e30ae,43013,1731715110098, seqNum=5 because the exception is null or not the one we care about 2024-11-16T00:01:24,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:24,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:24,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:24,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T00:01:24,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/445a6d40ac31495f823fe4ce56ea1771 2024-11-16T00:01:24,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d0bcdbc5c2b84d44824aae43b725cd13 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d0bcdbc5c2b84d44824aae43b725cd13 2024-11-16T00:01:24,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d0bcdbc5c2b84d44824aae43b725cd13, entries=150, sequenceid=386, filesize=30.5 K 2024-11-16T00:01:24,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/6480e05412d44edaa4af3b9dd55ad0bb as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6480e05412d44edaa4af3b9dd55ad0bb 2024-11-16T00:01:24,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6480e05412d44edaa4af3b9dd55ad0bb, entries=150, sequenceid=386, filesize=12.0 K 2024-11-16T00:01:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/445a6d40ac31495f823fe4ce56ea1771 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/445a6d40ac31495f823fe4ce56ea1771 2024-11-16T00:01:24,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/445a6d40ac31495f823fe4ce56ea1771, entries=150, sequenceid=386, filesize=12.0 K 2024-11-16T00:01:24,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 895ec0f83f32ef97e348b295ffa56e5a in 1483ms, sequenceid=386, compaction requested=false 2024-11-16T00:01:24,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:24,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-11-16T00:01:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:24,409 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T00:01:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166fb45b87d8694398a795b90a8462c621_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715282895/Put/seqid=0 2024-11-16T00:01:24,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742524_1700 (size=12454) 2024-11-16T00:01:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:24,823 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166fb45b87d8694398a795b90a8462c621_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166fb45b87d8694398a795b90a8462c621_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6ede017890864d5ab2b715e1fd24886f, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6ede017890864d5ab2b715e1fd24886f is 175, key is test_row_0/A:col10/1731715282895/Put/seqid=0 2024-11-16T00:01:24,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742525_1701 (size=31255) 2024-11-16T00:01:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43013 {}] regionserver.HRegion(8855): Flush requested on 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:25,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. as already flushing 2024-11-16T00:01:25,082 INFO [Thread-2617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2617. 2024-11-16T00:01:25,082 DEBUG [Thread-2617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:25,082 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 INFO [Thread-2619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2619. 2024-11-16T00:01:25,082 INFO [Thread-2615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2615. 2024-11-16T00:01:25,082 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 DEBUG [Thread-2619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:25,082 DEBUG [Thread-2615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:25,082 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 DEBUG [Thread-2615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:25,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:25,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:25,084 INFO [Thread-2611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2611. 2024-11-16T00:01:25,084 DEBUG [Thread-2611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:25,084 DEBUG [Thread-2611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,085 DEBUG [Thread-2611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:25,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:25,228 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6ede017890864d5ab2b715e1fd24886f 2024-11-16T00:01:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d75fe555d5cc4a35a15d0d2a9c423949 is 50, key is test_row_0/B:col10/1731715282895/Put/seqid=0 2024-11-16T00:01:25,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742526_1702 (size=12301) 2024-11-16T00:01:25,637 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d75fe555d5cc4a35a15d0d2a9c423949 2024-11-16T00:01:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/d89a3985595043639a69d64712ab6e1a is 50, key is test_row_0/C:col10/1731715282895/Put/seqid=0 2024-11-16T00:01:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742527_1703 (size=12301) 2024-11-16T00:01:26,046 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/d89a3985595043639a69d64712ab6e1a 2024-11-16T00:01:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/6ede017890864d5ab2b715e1fd24886f as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6ede017890864d5ab2b715e1fd24886f 2024-11-16T00:01:26,054 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6ede017890864d5ab2b715e1fd24886f, entries=150, sequenceid=403, filesize=30.5 K 2024-11-16T00:01:26,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/d75fe555d5cc4a35a15d0d2a9c423949 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d75fe555d5cc4a35a15d0d2a9c423949 2024-11-16T00:01:26,058 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d75fe555d5cc4a35a15d0d2a9c423949, entries=150, sequenceid=403, filesize=12.0 K 2024-11-16T00:01:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/d89a3985595043639a69d64712ab6e1a as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d89a3985595043639a69d64712ab6e1a 2024-11-16T00:01:26,062 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d89a3985595043639a69d64712ab6e1a, entries=150, sequenceid=403, filesize=12.0 K 2024-11-16T00:01:26,063 INFO [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=26.84 KB/27480 for 895ec0f83f32ef97e348b295ffa56e5a in 1654ms, sequenceid=403, compaction requested=true 2024-11-16T00:01:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for 895ec0f83f32ef97e348b295ffa56e5a: 2024-11-16T00:01:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:26,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0801813e30ae:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-11-16T00:01:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-11-16T00:01:26,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-16T00:01:26,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0870 sec 2024-11-16T00:01:26,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 3.1120 sec 2024-11-16T00:01:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-16T00:01:27,140 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:27,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T00:01:32,202 INFO [Thread-2613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2613. 2024-11-16T00:01:32,202 DEBUG [Thread-2613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T00:01:32,202 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:32,202 DEBUG [Thread-2613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:32,202 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2685 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2651 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2629 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2695 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2641 2024-11-16T00:01:32,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T00:01:32,203 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:01:32,203 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:01:32,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:32,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:32,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:32,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-16T00:01:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:32,206 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715292206"}]},"ts":"1731715292206"} 2024-11-16T00:01:32,208 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T00:01:32,208 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T00:01:32,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T00:01:32,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, UNASSIGN}] 2024-11-16T00:01:32,211 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, UNASSIGN 2024-11-16T00:01:32,212 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=CLOSING, regionLocation=0801813e30ae,43013,1731715110098 2024-11-16T00:01:32,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, UNASSIGN because future has completed 2024-11-16T00:01:32,213 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T00:01:32,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098}] 2024-11-16T00:01:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:32,365 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(122): Close 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:32,365 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T00:01:32,365 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1722): Closing 895ec0f83f32ef97e348b295ffa56e5a, disabling compactions & flushes 2024-11-16T00:01:32,365 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:32,365 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:32,365 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. after waiting 0 ms 2024-11-16T00:01:32,365 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:32,365 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(2902): Flushing 895ec0f83f32ef97e348b295ffa56e5a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T00:01:32,370 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116456742046fd248188bd731801b058034_895ec0f83f32ef97e348b295ffa56e5a is 50, key is test_row_0/A:col10/1731715292200/Put/seqid=0 2024-11-16T00:01:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742528_1704 (size=9914) 2024-11-16T00:01:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:32,774 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T00:01:32,777 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116456742046fd248188bd731801b058034_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116456742046fd248188bd731801b058034_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:32,778 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d7a35627620e426fbe5ee2a433544fd4, store: [table=TestAcidGuarantees family=A region=895ec0f83f32ef97e348b295ffa56e5a] 2024-11-16T00:01:32,778 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d7a35627620e426fbe5ee2a433544fd4 is 175, key is test_row_0/A:col10/1731715292200/Put/seqid=0 2024-11-16T00:01:32,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742529_1705 (size=22561) 2024-11-16T00:01:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:33,181 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d7a35627620e426fbe5ee2a433544fd4 2024-11-16T00:01:33,187 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/8b33f7afab434a69bfcd23b322e7a1cd is 50, key is test_row_0/B:col10/1731715292200/Put/seqid=0 2024-11-16T00:01:33,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742530_1706 (size=9857) 2024-11-16T00:01:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:33,590 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/8b33f7afab434a69bfcd23b322e7a1cd 2024-11-16T00:01:33,595 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1eeb58d89d774aae8779e606abf010a1 is 50, key is test_row_0/C:col10/1731715292200/Put/seqid=0 2024-11-16T00:01:33,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742531_1707 (size=9857) 2024-11-16T00:01:33,999 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1eeb58d89d774aae8779e606abf010a1 2024-11-16T00:01:34,002 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/A/d7a35627620e426fbe5ee2a433544fd4 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d7a35627620e426fbe5ee2a433544fd4 2024-11-16T00:01:34,004 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d7a35627620e426fbe5ee2a433544fd4, entries=100, sequenceid=411, filesize=22.0 K 2024-11-16T00:01:34,005 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/B/8b33f7afab434a69bfcd23b322e7a1cd as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/8b33f7afab434a69bfcd23b322e7a1cd 2024-11-16T00:01:34,007 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/8b33f7afab434a69bfcd23b322e7a1cd, entries=100, sequenceid=411, filesize=9.6 K 2024-11-16T00:01:34,008 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/.tmp/C/1eeb58d89d774aae8779e606abf010a1 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1eeb58d89d774aae8779e606abf010a1 2024-11-16T00:01:34,010 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1eeb58d89d774aae8779e606abf010a1, entries=100, sequenceid=411, filesize=9.6 K 2024-11-16T00:01:34,011 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 895ec0f83f32ef97e348b295ffa56e5a in 1646ms, sequenceid=411, compaction requested=true 2024-11-16T00:01:34,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8] to archive 2024-11-16T00:01:34,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:34,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7142e6b8af4c40e1ad8b4fd0b2a52485 2024-11-16T00:01:34,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d16fb81e75af4ac1be536759ff3fb7c5 2024-11-16T00:01:34,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0e8e4d6e08f418c93b080d0ac58ba4b 2024-11-16T00:01:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6b117e4829b94a2dbacfb8836c51293d 2024-11-16T00:01:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/9f5cf49fdd8847489b837805baebb48d 2024-11-16T00:01:34,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4b9aac03a3ab44a99157875b1243732c 2024-11-16T00:01:34,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/bc9809ac0b314e299dab519d271d0cf1 2024-11-16T00:01:34,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/cbcab3aee0e743f0ad4399745bcc595a 2024-11-16T00:01:34,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2546c1cbd7e84537b72f5a18f3574b14 2024-11-16T00:01:34,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/4bd6f956492b41c69a49328ffb49dbab 2024-11-16T00:01:34,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/5b084555c2d243f0a29fda9c8e5ff4d5 2024-11-16T00:01:34,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/e309a885f4b040d99a5044d1f41f48f6 2024-11-16T00:01:34,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/63889eafc536418197f83a3eea5f8e8a 2024-11-16T00:01:34,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/b0b8dfaa75e546809f28bc5fc32cc275 2024-11-16T00:01:34,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/942b897d2928474ab9e27d799bb12e8f 2024-11-16T00:01:34,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/fd12fd1dbb9f4f1a8e6a086f52dc00cf 2024-11-16T00:01:34,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/24110c685a2e4607a43e027756c6876d 2024-11-16T00:01:34,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d533fe3b834640daac955d8765e88119 2024-11-16T00:01:34,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/2e1b2225d5134e01972d786c97abc85c 2024-11-16T00:01:34,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/477716f102004bf380a271f42bc91f86 2024-11-16T00:01:34,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/c7e4a3e098f046cf917796ca6d0e75be 2024-11-16T00:01:34,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/dba300b487b0475497d1c4d0c9cc1877 2024-11-16T00:01:34,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/f5433fc7ef1f4df3bb4312465ae6102d 2024-11-16T00:01:34,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/0d57a21a9196476088acd3abf0f1b40f 2024-11-16T00:01:34,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/74e2abcede57486299bfc2bb99a427f8 2024-11-16T00:01:34,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4554718f914845988c68c1ec8c8eba9c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2806aef1af3e44578ad40f07e02f05c7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5cbacd7ec6954eefa039734113c5ef04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/048eb6a515d14e6897b711fa158e3bf1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9b82fb37cdda4719b010d4452b7bef75, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/46725def8844401e8b5648243885f7f9, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee] to archive 2024-11-16T00:01:34,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:34,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b52909e142c49a5b93f35f68d669a85 2024-11-16T00:01:34,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/03aefa808fa44e18a19f2b6333a7ed67 2024-11-16T00:01:34,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4554718f914845988c68c1ec8c8eba9c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4554718f914845988c68c1ec8c8eba9c 2024-11-16T00:01:34,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/cea692b80ee14126b20311778f7f8200 2024-11-16T00:01:34,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ca95584297d44cadac51e0a481551fc3 2024-11-16T00:01:34,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4a95d8cb79f24c5bbef803bc02894dbe 2024-11-16T00:01:34,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/09e5a38d3ece422ca5fd97b9b2923578 2024-11-16T00:01:34,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/a4b1d736a97144699548dfe8385c5bcc 2024-11-16T00:01:34,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2806aef1af3e44578ad40f07e02f05c7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2806aef1af3e44578ad40f07e02f05c7 2024-11-16T00:01:34,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5486427da0364db1ac1b92d7c4579ce6 2024-11-16T00:01:34,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d656f8ddaa5f4587aec508eb8d985e69 2024-11-16T00:01:34,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/740f29bcc9374c16bb8c6f8d465b05c0 2024-11-16T00:01:34,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5cbacd7ec6954eefa039734113c5ef04 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/5cbacd7ec6954eefa039734113c5ef04 2024-11-16T00:01:34,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f37767fc59b140bf95b831d45fbacdea 2024-11-16T00:01:34,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/f8a5fe77caf54474bf99c42842effcb4 2024-11-16T00:01:34,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/048eb6a515d14e6897b711fa158e3bf1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/048eb6a515d14e6897b711fa158e3bf1 2024-11-16T00:01:34,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6383cc5deb1442048e4126c4b84df8c1 2024-11-16T00:01:34,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/519137d4203848ec802dd5bc79ce0381 2024-11-16T00:01:34,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9b82fb37cdda4719b010d4452b7bef75 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9b82fb37cdda4719b010d4452b7bef75 2024-11-16T00:01:34,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/ad5de1fea1194050971c7e82a25f274f 2024-11-16T00:01:34,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/e1a30d6de1f94d3799e78e5dadd18943 2024-11-16T00:01:34,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/46725def8844401e8b5648243885f7f9 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/46725def8844401e8b5648243885f7f9 2024-11-16T00:01:34,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/4b7220d2575341ada6a0b56459393edc 2024-11-16T00:01:34,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/15be7b27d4e04d51a4b5012602560f0c 2024-11-16T00:01:34,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/2a03b94ad0474a0ca050d5b480e51aee 2024-11-16T00:01:34,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d57ad5141a714e70a96f78bb70986290, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/07b0ef5c84f146c7a91f3b28b4f43060, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/57ce14a46bad4afaa3fb8911d68375a2, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/10e327444aad4afea9d2965dd187ef8c, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/18c8a37fa77b4dd78945152cccf7e278, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/fcebfe0a588a4758be9a21c1d698076d, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca] to archive 2024-11-16T00:01:34,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T00:01:34,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/45d8d2fce5dd4e939d2db121c957b46b 2024-11-16T00:01:34,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/50b26a7f4f2c48e1b4fe5289e65cd488 2024-11-16T00:01:34,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d57ad5141a714e70a96f78bb70986290 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d57ad5141a714e70a96f78bb70986290 2024-11-16T00:01:34,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/85db7999fc1b4a49879388aa9714e198 2024-11-16T00:01:34,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/a2ba0d3bdcf6447faeaf00588c608f41 2024-11-16T00:01:34,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f89bab52157e4b4eb551a538d9eda226 2024-11-16T00:01:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/cc3c94b0ae2241beadbad6e3125ed819 2024-11-16T00:01:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/4d3b7179072647e2932244f1c65e3996 2024-11-16T00:01:34,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/07b0ef5c84f146c7a91f3b28b4f43060 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/07b0ef5c84f146c7a91f3b28b4f43060 2024-11-16T00:01:34,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/13b9e2f3627b45299ef0d3c9c18a39db 2024-11-16T00:01:34,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/0c39d50f2c3840cb9e2f6d6605367a58 2024-11-16T00:01:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3cee8bb6d81b4964a995946c32256867 2024-11-16T00:01:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/57ce14a46bad4afaa3fb8911d68375a2 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/57ce14a46bad4afaa3fb8911d68375a2 2024-11-16T00:01:34,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/5f4f44820b974620bd5a025234b33a1e 2024-11-16T00:01:34,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/f6ef8e85918c48348f15e79ce00ea8e7 2024-11-16T00:01:34,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/10e327444aad4afea9d2965dd187ef8c to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/10e327444aad4afea9d2965dd187ef8c 2024-11-16T00:01:34,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/c469351a77284f0f9623a371853e8083 2024-11-16T00:01:34,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/9f813f0272564eea84f9e5feac443c26 2024-11-16T00:01:34,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/18c8a37fa77b4dd78945152cccf7e278 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/18c8a37fa77b4dd78945152cccf7e278 2024-11-16T00:01:34,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/993690333e7244a8b8d88751800ce43b 2024-11-16T00:01:34,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/7483107449a046febf5ea9f61692e8ad 2024-11-16T00:01:34,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/fcebfe0a588a4758be9a21c1d698076d to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/fcebfe0a588a4758be9a21c1d698076d 2024-11-16T00:01:34,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2462585e471b4f0c8b9c7911a5430e04 2024-11-16T00:01:34,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1111b1dcc4194f6fa5d9947374657496 2024-11-16T00:01:34,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/3e2061943af3411287960a61dba90fca 2024-11-16T00:01:34,078 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits/414.seqid, newMaxSeqId=414, maxSeqId=4 2024-11-16T00:01:34,079 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. 2024-11-16T00:01:34,079 DEBUG [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1676): Region close journal for 895ec0f83f32ef97e348b295ffa56e5a: Waiting for close lock at 1731715292365Running coprocessor pre-close hooks at 1731715292365Disabling compacts and flushes for region at 1731715292365Disabling writes for close at 1731715292365Obtaining lock to block concurrent updates at 1731715292365Preparing flush snapshotting stores in 895ec0f83f32ef97e348b295ffa56e5a at 1731715292365Finished memstore snapshotting TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731715292366 (+1 ms)Flushing stores of TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a. at 1731715292366Flushing 895ec0f83f32ef97e348b295ffa56e5a/A: creating writer at 1731715292366Flushing mob file 895ec0f83f32ef97e348b295ffa56e5a/A: appending metadata at 1731715292370 (+4 ms)Flushing mob file 895ec0f83f32ef97e348b295ffa56e5a/A: closing flushed file at 1731715292370Flushing 895ec0f83f32ef97e348b295ffa56e5a/A: appending metadata at 1731715292778 (+408 ms)Flushing 895ec0f83f32ef97e348b295ffa56e5a/A: closing flushed file at 1731715292778Flushing 895ec0f83f32ef97e348b295ffa56e5a/B: creating writer at 1731715293185 (+407 ms)Flushing 895ec0f83f32ef97e348b295ffa56e5a/B: appending metadata at 1731715293186 (+1 ms)Flushing 895ec0f83f32ef97e348b295ffa56e5a/B: closing flushed file at 1731715293186Flushing 895ec0f83f32ef97e348b295ffa56e5a/C: creating writer at 1731715293593 (+407 ms)Flushing 895ec0f83f32ef97e348b295ffa56e5a/C: appending metadata at 1731715293595 (+2 ms)Flushing 895ec0f83f32ef97e348b295ffa56e5a/C: closing flushed file at 1731715293595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57b8180e: reopening flushed file at 1731715294002 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ed4dd7f: reopening flushed file at 1731715294004 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38a5f9ae: reopening flushed file at 1731715294007 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 895ec0f83f32ef97e348b295ffa56e5a in 1646ms, sequenceid=411, compaction requested=true at 1731715294011 (+4 ms)Writing region close event to WAL at 1731715294076 (+65 ms)Running coprocessor post-close hooks at 1731715294079 (+3 ms)Closed at 1731715294079 2024-11-16T00:01:34,081 INFO [RS_CLOSE_REGION-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(157): Closed 895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,081 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=895ec0f83f32ef97e348b295ffa56e5a, regionState=CLOSED 2024-11-16T00:01:34,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 because future has completed 2024-11-16T00:01:34,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-11-16T00:01:34,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; CloseRegionProcedure 895ec0f83f32ef97e348b295ffa56e5a, server=0801813e30ae,43013,1731715110098 in 1.8710 sec 2024-11-16T00:01:34,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-11-16T00:01:34,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=895ec0f83f32ef97e348b295ffa56e5a, UNASSIGN in 1.8760 sec 2024-11-16T00:01:34,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-16T00:01:34,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-16T00:01:34,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731715294091"}]},"ts":"1731715294091"} 2024-11-16T00:01:34,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T00:01:34,092 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T00:01:34,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-16T00:01:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-16T00:01:34,350 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:34,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-16T00:01:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,352 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-16T00:01:34,352 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=186, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,354 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,356 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C, FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits] 2024-11-16T00:01:34,358 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6ede017890864d5ab2b715e1fd24886f to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/6ede017890864d5ab2b715e1fd24886f 2024-11-16T00:01:34,359 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7885e4f9d41d4b20b6c782df54dff123 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/7885e4f9d41d4b20b6c782df54dff123 2024-11-16T00:01:34,360 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d0bcdbc5c2b84d44824aae43b725cd13 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d0bcdbc5c2b84d44824aae43b725cd13 2024-11-16T00:01:34,360 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d7a35627620e426fbe5ee2a433544fd4 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/A/d7a35627620e426fbe5ee2a433544fd4 2024-11-16T00:01:34,362 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6480e05412d44edaa4af3b9dd55ad0bb to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/6480e05412d44edaa4af3b9dd55ad0bb 2024-11-16T00:01:34,363 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/8b33f7afab434a69bfcd23b322e7a1cd to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/8b33f7afab434a69bfcd23b322e7a1cd 2024-11-16T00:01:34,364 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9fe924d5a4204b058f54cff9f5d6b4ce to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/9fe924d5a4204b058f54cff9f5d6b4ce 2024-11-16T00:01:34,365 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d75fe555d5cc4a35a15d0d2a9c423949 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/B/d75fe555d5cc4a35a15d0d2a9c423949 2024-11-16T00:01:34,367 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1eeb58d89d774aae8779e606abf010a1 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/1eeb58d89d774aae8779e606abf010a1 2024-11-16T00:01:34,368 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2ad4d418361b4e298ccceb0ffe23d360 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/2ad4d418361b4e298ccceb0ffe23d360 2024-11-16T00:01:34,369 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/445a6d40ac31495f823fe4ce56ea1771 to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/445a6d40ac31495f823fe4ce56ea1771 2024-11-16T00:01:34,369 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d89a3985595043639a69d64712ab6e1a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/C/d89a3985595043639a69d64712ab6e1a 2024-11-16T00:01:34,372 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits/414.seqid to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a/recovered.edits/414.seqid 2024-11-16T00:01:34,372 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/default/TestAcidGuarantees/895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,372 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T00:01:34,372 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T00:01:34,373 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T00:01:34,375 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116025dc14ddabb4e849df5366cc6942283_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116025dc14ddabb4e849df5366cc6942283_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602e8d5a45c20473da5b41d699a19561f_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602e8d5a45c20473da5b41d699a19561f_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,377 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7429c080984240b22bd4013eb85b4e_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160b7429c080984240b22bd4013eb85b4e_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,378 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161297647efa8744ba8d2cd69bab8c5e87_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161297647efa8744ba8d2cd69bab8c5e87_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,379 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162779f69228dc4d1cb9eeb686af91b317_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162779f69228dc4d1cb9eeb686af91b317_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,380 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116295fafbe373d4657a86ed663b3ab4af9_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116295fafbe373d4657a86ed663b3ab4af9_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,381 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116456742046fd248188bd731801b058034_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116456742046fd248188bd731801b058034_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,382 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165497ad65712846ca946f752664d945e1_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165497ad65712846ca946f752664d945e1_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec940bbce964f7ca9ceb3cfb501fee4_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec940bbce964f7ca9ceb3cfb501fee4_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665b135da3c104982bedbe695217909f5_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665b135da3c104982bedbe695217909f5_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,384 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116679ec266fdf14f1aa23634158a56502a_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116679ec266fdf14f1aa23634158a56502a_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,385 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f496279f27d41a8a3acdc261d385739_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166f496279f27d41a8a3acdc261d385739_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,386 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166fb45b87d8694398a795b90a8462c621_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166fb45b87d8694398a795b90a8462c621_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,387 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116817c2fd694ca4c388b81121e369e3746_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116817c2fd694ca4c388b81121e369e3746_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686d55948501540f6a67f013632450197_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686d55948501540f6a67f013632450197_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,389 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f1ec153dcc245a8ad7e6d7b6dcdd68f_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f1ec153dcc245a8ad7e6d7b6dcdd68f_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,389 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b075791a0e9643eeb39a78214f3206cc_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b075791a0e9643eeb39a78214f3206cc_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,390 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c64e07bbc5924280917edbf9cc7e04ab_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c64e07bbc5924280917edbf9cc7e04ab_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,391 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cc36c98597de4f97a188f9663986eed5_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cc36c98597de4f97a188f9663986eed5_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,392 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f46a1759f39240fbb4cc1f81b2f1f259_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f46a1759f39240fbb4cc1f81b2f1f259_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,393 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f88eff79bf6441ec9e1edd32a32531bc_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f88eff79bf6441ec9e1edd32a32531bc_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,394 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fad08052fedb474080d64d0ead760445_895ec0f83f32ef97e348b295ffa56e5a to hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fad08052fedb474080d64d0ead760445_895ec0f83f32ef97e348b295ffa56e5a 2024-11-16T00:01:34,394 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T00:01:34,396 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=186, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,398 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T00:01:34,400 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T00:01:34,401 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=186, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,401 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T00:01:34,402 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731715294401"}]},"ts":"9223372036854775807"} 2024-11-16T00:01:34,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T00:01:34,403 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 895ec0f83f32ef97e348b295ffa56e5a, NAME => 'TestAcidGuarantees,,1731715260702.895ec0f83f32ef97e348b295ffa56e5a.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T00:01:34,403 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T00:01:34,404 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731715294403"}]},"ts":"9223372036854775807"} 2024-11-16T00:01:34,405 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T00:01:34,406 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=186, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T00:01:34,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-16T00:01:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-11-16T00:01:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T00:01:34,461 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T00:01:34,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236 (was 239), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 666), ProcessCount=11 (was 11), AvailableMemoryMB=4114 (was 4468) 2024-11-16T00:01:34,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T00:01:34,470 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:01:34,470 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:01:34,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:34,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:34,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T00:01:34,470 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T00:01:34,470 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117780854, stopped=false 2024-11-16T00:01:34,470 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0801813e30ae,38967,1731715109050 2024-11-16T00:01:34,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T00:01:34,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T00:01:34,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T00:01:34,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T00:01:34,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T00:01:34,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T00:01:34,473 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T00:01:34,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T00:01:34,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:01:34,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:34,474 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0801813e30ae,43013,1731715110098' ***** 2024-11-16T00:01:34,474 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T00:01:34,475 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(959): stopping server 0801813e30ae,43013,1731715110098 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T00:01:34,475 INFO [RS:0;0801813e30ae:43013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0801813e30ae:43013. 2024-11-16T00:01:34,475 DEBUG [RS:0;0801813e30ae:43013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T00:01:34,475 DEBUG [RS:0;0801813e30ae:43013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:34,476 INFO [RS:0;0801813e30ae:43013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T00:01:34,476 INFO [RS:0;0801813e30ae:43013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T00:01:34,476 INFO [RS:0;0801813e30ae:43013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T00:01:34,476 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T00:01:34,476 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T00:01:34,476 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T00:01:34,477 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T00:01:34,477 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:34,477 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T00:01:34,477 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T00:01:34,477 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T00:01:34,477 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T00:01:34,477 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T00:01:34,498 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/info/b8983a6d89be4ddcb5e5bfe92f009f03 is 95, key is TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c./info:/1731715142130/DeleteFamily/seqid=0 2024-11-16T00:01:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742532_1708 (size=6514) 2024-11-16T00:01:34,519 INFO [regionserver/0801813e30ae:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T00:01:34,677 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:34,863 INFO [regionserver/0801813e30ae:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T00:01:34,863 INFO [regionserver/0801813e30ae:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T00:01:34,877 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:34,901 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/info/b8983a6d89be4ddcb5e5bfe92f009f03 2024-11-16T00:01:34,919 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/ns/f9c2e1c1c8314ac689ced577d36396f8 is 93, key is TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c./ns:/1731715142068/DeleteFamily/seqid=0 2024-11-16T00:01:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742533_1709 (size=5906) 2024-11-16T00:01:35,077 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:35,277 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:35,323 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/ns/f9c2e1c1c8314ac689ced577d36396f8 2024-11-16T00:01:35,340 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/rep_barrier/f7a1ada9afe44e7588f4fbf1bf255c81 is 102, key is TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c./rep_barrier:/1731715142068/DeleteFamily/seqid=0 2024-11-16T00:01:35,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742534_1710 (size=6025) 2024-11-16T00:01:35,478 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:35,678 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:35,743 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/rep_barrier/f7a1ada9afe44e7588f4fbf1bf255c81 2024-11-16T00:01:35,761 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/table/33f9c55f17ae4b8abdaed6e9771d6dea is 96, key is TestAcidGuarantees,,1731715114628.131fcc9fbcad104b408318bd0c6e140c./table:/1731715142068/DeleteFamily/seqid=0 2024-11-16T00:01:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742535_1711 (size=5936) 2024-11-16T00:01:35,878 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:36,078 DEBUG [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T00:01:36,165 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/table/33f9c55f17ae4b8abdaed6e9771d6dea 2024-11-16T00:01:36,169 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/info/b8983a6d89be4ddcb5e5bfe92f009f03 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/info/b8983a6d89be4ddcb5e5bfe92f009f03 2024-11-16T00:01:36,172 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/info/b8983a6d89be4ddcb5e5bfe92f009f03, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T00:01:36,173 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/ns/f9c2e1c1c8314ac689ced577d36396f8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/ns/f9c2e1c1c8314ac689ced577d36396f8 2024-11-16T00:01:36,177 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/ns/f9c2e1c1c8314ac689ced577d36396f8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T00:01:36,177 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/rep_barrier/f7a1ada9afe44e7588f4fbf1bf255c81 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/rep_barrier/f7a1ada9afe44e7588f4fbf1bf255c81 2024-11-16T00:01:36,181 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/rep_barrier/f7a1ada9afe44e7588f4fbf1bf255c81, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T00:01:36,182 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/.tmp/table/33f9c55f17ae4b8abdaed6e9771d6dea as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/table/33f9c55f17ae4b8abdaed6e9771d6dea 2024-11-16T00:01:36,185 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/table/33f9c55f17ae4b8abdaed6e9771d6dea, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T00:01:36,186 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-11-16T00:01:36,190 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T00:01:36,191 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T00:01:36,191 INFO [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T00:01:36,191 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731715294476Running coprocessor pre-close hooks at 1731715294477 (+1 ms)Disabling compacts and flushes for region at 1731715294477Disabling writes for close at 1731715294477Obtaining lock to block concurrent updates at 1731715294477Preparing flush snapshotting stores in 1588230740 at 1731715294477Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731715294477Flushing stores of hbase:meta,,1.1588230740 at 1731715294478 (+1 ms)Flushing 1588230740/info: creating writer at 1731715294478Flushing 1588230740/info: appending metadata at 1731715294497 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731715294497Flushing 1588230740/ns: creating writer at 1731715294905 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731715294919 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731715294919Flushing 1588230740/rep_barrier: creating writer at 1731715295326 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731715295340 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731715295340Flushing 1588230740/table: creating writer at 1731715295746 (+406 ms)Flushing 1588230740/table: appending metadata at 1731715295761 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731715295761Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27a4bfd7: reopening flushed file at 1731715296168 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4913cd67: reopening flushed file at 1731715296173 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@122bc6f8: reopening flushed file at 1731715296177 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@415b095: reopening flushed file at 1731715296181 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1731715296186 (+5 ms)Writing region close event to WAL at 1731715296187 (+1 ms)Running coprocessor post-close hooks at 1731715296191 (+4 ms)Closed at 1731715296191 2024-11-16T00:01:36,191 DEBUG [RS_CLOSE_META-regionserver/0801813e30ae:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T00:01:36,279 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(976): stopping server 0801813e30ae,43013,1731715110098; all regions closed. 2024-11-16T00:01:36,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741834_1010 (size=25344) 2024-11-16T00:01:36,285 DEBUG [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/oldWALs 2024-11-16T00:01:36,285 INFO [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0801813e30ae%2C43013%2C1731715110098.meta:.meta(num 1731715114037) 2024-11-16T00:01:36,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741832_1008 (size=16146913) 2024-11-16T00:01:36,288 DEBUG [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/oldWALs 2024-11-16T00:01:36,288 INFO [RS:0;0801813e30ae:43013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0801813e30ae%2C43013%2C1731715110098:(num 1731715113040) 2024-11-16T00:01:36,288 DEBUG [RS:0;0801813e30ae:43013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T00:01:36,288 INFO [RS:0;0801813e30ae:43013 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T00:01:36,288 INFO [RS:0;0801813e30ae:43013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T00:01:36,289 INFO [RS:0;0801813e30ae:43013 {}] hbase.ChoreService(370): Chore service for: regionserver/0801813e30ae:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T00:01:36,289 INFO [RS:0;0801813e30ae:43013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T00:01:36,289 INFO [regionserver/0801813e30ae:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T00:01:36,289 INFO [RS:0;0801813e30ae:43013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43013 2024-11-16T00:01:36,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0801813e30ae,43013,1731715110098 2024-11-16T00:01:36,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T00:01:36,293 INFO [RS:0;0801813e30ae:43013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T00:01:36,294 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0801813e30ae,43013,1731715110098] 2024-11-16T00:01:36,296 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0801813e30ae,43013,1731715110098 already deleted, retry=false 2024-11-16T00:01:36,296 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0801813e30ae,43013,1731715110098 expired; onlineServers=0 2024-11-16T00:01:36,297 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0801813e30ae,38967,1731715109050' ***** 2024-11-16T00:01:36,297 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T00:01:36,297 INFO [M:0;0801813e30ae:38967 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T00:01:36,297 INFO [M:0;0801813e30ae:38967 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T00:01:36,297 DEBUG [M:0;0801813e30ae:38967 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T00:01:36,297 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T00:01:36,297 DEBUG [M:0;0801813e30ae:38967 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T00:01:36,297 DEBUG [master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.large.0-1731715112727 {}] cleaner.HFileCleaner(306): Exit Thread[master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.large.0-1731715112727,5,FailOnTimeoutGroup] 2024-11-16T00:01:36,297 DEBUG [master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.small.0-1731715112737 {}] cleaner.HFileCleaner(306): Exit Thread[master/0801813e30ae:0:becomeActiveMaster-HFileCleaner.small.0-1731715112737,5,FailOnTimeoutGroup] 2024-11-16T00:01:36,297 INFO [M:0;0801813e30ae:38967 {}] hbase.ChoreService(370): Chore service for: master/0801813e30ae:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T00:01:36,297 INFO [M:0;0801813e30ae:38967 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T00:01:36,298 DEBUG [M:0;0801813e30ae:38967 {}] master.HMaster(1795): Stopping service threads 2024-11-16T00:01:36,298 INFO [M:0;0801813e30ae:38967 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T00:01:36,298 INFO [M:0;0801813e30ae:38967 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T00:01:36,298 ERROR [M:0;0801813e30ae:38967 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T00:01:36,298 INFO [M:0;0801813e30ae:38967 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T00:01:36,298 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T00:01:36,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T00:01:36,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T00:01:36,299 DEBUG [M:0;0801813e30ae:38967 {}] zookeeper.ZKUtil(347): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T00:01:36,299 WARN [M:0;0801813e30ae:38967 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T00:01:36,299 INFO [M:0;0801813e30ae:38967 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/.lastflushedseqids 2024-11-16T00:01:36,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742536_1712 (size=72) 2024-11-16T00:01:36,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T00:01:36,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x100f8d681700001, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T00:01:36,396 INFO [RS:0;0801813e30ae:43013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T00:01:36,397 INFO [RS:0;0801813e30ae:43013 {}] regionserver.HRegionServer(1031): Exiting; stopping=0801813e30ae,43013,1731715110098; zookeeper connection closed. 2024-11-16T00:01:36,397 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@707bf8da {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@707bf8da 2024-11-16T00:01:36,398 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T00:01:36,709 INFO [M:0;0801813e30ae:38967 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T00:01:36,709 INFO [M:0;0801813e30ae:38967 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T00:01:36,710 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T00:01:36,710 INFO [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T00:01:36,710 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T00:01:36,710 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T00:01:36,710 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T00:01:36,710 INFO [M:0;0801813e30ae:38967 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=791.80 KB heapSize=975.80 KB 2024-11-16T00:01:36,725 DEBUG [M:0;0801813e30ae:38967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59e88195e8814b04b2f1da0addec6c74 is 82, key is hbase:meta,,1/info:regioninfo/1731715114199/Put/seqid=0 2024-11-16T00:01:36,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742537_1713 (size=5672) 2024-11-16T00:01:37,129 INFO [M:0;0801813e30ae:38967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2268 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59e88195e8814b04b2f1da0addec6c74 2024-11-16T00:01:37,149 DEBUG [M:0;0801813e30ae:38967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5eafe24db33d45e1ac62a4c42c40dd18 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x99/proc:d/1731715263773/Put/seqid=0 2024-11-16T00:01:37,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742538_1714 (size=47004) 2024-11-16T00:01:37,555 INFO [M:0;0801813e30ae:38967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=791.24 KB at sequenceid=2268 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5eafe24db33d45e1ac62a4c42c40dd18 2024-11-16T00:01:37,558 INFO [M:0;0801813e30ae:38967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5eafe24db33d45e1ac62a4c42c40dd18 2024-11-16T00:01:37,573 DEBUG [M:0;0801813e30ae:38967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7db6a37bf7344dd1a5383e96f47eccd8 is 69, key is 0801813e30ae,43013,1731715110098/rs:state/1731715112762/Put/seqid=0 2024-11-16T00:01:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073742539_1715 (size=5156) 2024-11-16T00:01:37,977 INFO [M:0;0801813e30ae:38967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2268 (bloomFilter=true), to=hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7db6a37bf7344dd1a5383e96f47eccd8 2024-11-16T00:01:37,980 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/59e88195e8814b04b2f1da0addec6c74 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/59e88195e8814b04b2f1da0addec6c74 2024-11-16T00:01:37,983 INFO [M:0;0801813e30ae:38967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/59e88195e8814b04b2f1da0addec6c74, entries=8, sequenceid=2268, filesize=5.5 K 2024-11-16T00:01:37,983 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5eafe24db33d45e1ac62a4c42c40dd18 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5eafe24db33d45e1ac62a4c42c40dd18 2024-11-16T00:01:37,986 INFO [M:0;0801813e30ae:38967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5eafe24db33d45e1ac62a4c42c40dd18 2024-11-16T00:01:37,986 INFO [M:0;0801813e30ae:38967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5eafe24db33d45e1ac62a4c42c40dd18, entries=186, sequenceid=2268, filesize=45.9 K 2024-11-16T00:01:37,986 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7db6a37bf7344dd1a5383e96f47eccd8 as hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7db6a37bf7344dd1a5383e96f47eccd8 2024-11-16T00:01:37,988 INFO [M:0;0801813e30ae:38967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44829/user/jenkins/test-data/7d3ce901-2899-cdb1-eaf3-694f3190c53e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7db6a37bf7344dd1a5383e96f47eccd8, entries=1, sequenceid=2268, filesize=5.0 K 2024-11-16T00:01:37,989 INFO [M:0;0801813e30ae:38967 {}] regionserver.HRegion(3140): Finished flush of dataSize ~791.80 KB/810803, heapSize ~975.50 KB/998912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2268, compaction requested=false 2024-11-16T00:01:37,990 INFO [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T00:01:37,990 DEBUG [M:0;0801813e30ae:38967 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731715296709Disabling compacts and flushes for region at 1731715296709Disabling writes for close at 1731715296710 (+1 ms)Obtaining lock to block concurrent updates at 1731715296710Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731715296710Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=810803, getHeapSize=999152, getOffHeapSize=0, getCellsCount=2479 at 1731715296710Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731715296711 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731715296711Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731715296725 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731715296725Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731715297132 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731715297148 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731715297148Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731715297559 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731715297573 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731715297573Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22aba89f: reopening flushed file at 1731715297980 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a3b1e79: reopening flushed file at 1731715297983 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c0af79: reopening flushed file at 1731715297986 (+3 ms)Finished flush of dataSize ~791.80 KB/810803, heapSize ~975.50 KB/998912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2268, compaction requested=false at 1731715297989 (+3 ms)Writing region close event to WAL at 1731715297990 (+1 ms)Closed at 1731715297990 2024-11-16T00:01:37,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43069 is added to blk_1073741830_1006 (size=959486) 2024-11-16T00:01:37,993 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T00:01:37,993 INFO [M:0;0801813e30ae:38967 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T00:01:37,993 INFO [M:0;0801813e30ae:38967 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38967 2024-11-16T00:01:37,994 INFO [M:0;0801813e30ae:38967 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T00:01:38,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T00:01:38,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38967-0x100f8d681700000, quorum=127.0.0.1:61510, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T00:01:38,096 INFO [M:0;0801813e30ae:38967 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T00:01:38,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15254365{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T00:01:38,103 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ae5a080{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T00:01:38,103 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T00:01:38,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T00:01:38,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.log.dir/,STOPPED} 2024-11-16T00:01:38,106 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T00:01:38,106 WARN [BP-1500091864-172.17.0.3-1731715105508 heartbeating to localhost/127.0.0.1:44829 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T00:01:38,107 WARN [BP-1500091864-172.17.0.3-1731715105508 heartbeating to localhost/127.0.0.1:44829 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1500091864-172.17.0.3-1731715105508 (Datanode Uuid 674630d6-a271-46f8-906d-fa5228c2c352) service to localhost/127.0.0.1:44829 2024-11-16T00:01:38,107 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T00:01:38,109 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/data/data1/current/BP-1500091864-172.17.0.3-1731715105508 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T00:01:38,109 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/cluster_03ce3908-b22f-78aa-beb5-e4988fd6bb7f/data/data2/current/BP-1500091864-172.17.0.3-1731715105508 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T00:01:38,110 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T00:01:38,117 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T00:01:38,118 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T00:01:38,118 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T00:01:38,118 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T00:01:38,118 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0484c88-4e17-d9a3-7c39-980a684e835b/hadoop.log.dir/,STOPPED} 2024-11-16T00:01:38,142 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T00:01:38,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down